diff --git a/docs/plugins/writing-a-plugin.md b/docs/plugins/writing-a-plugin.md index 5653414..19d5348 100644 --- a/docs/plugins/writing-a-plugin.md +++ b/docs/plugins/writing-a-plugin.md @@ -323,6 +323,21 @@ Role hierarchy: `admin > operator > viewer`. Requiring `viewer` grants access to --- +## Audit Log Entries + +A change made through a plugin's pages is audited with `steward.core.audit.log_audit`, naming the thing it changed by `entity_type` and `entity_id`. So the audit page shows that thing's current name, linked to its page, export `get_audit_entities()` from `__init__.py`: + +```python +# plugins/myplugin/__init__.py +def get_audit_entities() -> dict: + from .models import Target + return {"myplugin_target": (Target, "/plugins/myplugin/targets/{}")} +``` + +The model needs `id` and `name` columns. An entity deleted since keeps the name the entry recorded in its detail (`detail={"name": ...}`). Add the action's wording to `ACTIONS` in `steward/audit/service.py` so the row reads as a sentence. + +--- + ## Agent Tools (MCP) Steward serves an MCP endpoint at `/mcp` that AI clients reach with a personal API token created on the AI access page. A plugin adds tools to it by exporting an optional `register_mcp_tools(register)` from `__init__.py`: diff --git a/plugins/snmp/__init__.py b/plugins/snmp/__init__.py index e43d288..1aa6228 100644 --- a/plugins/snmp/__init__.py +++ b/plugins/snmp/__init__.py @@ -11,6 +11,7 @@ _app: "Quart | None" = None def setup(app: "Quart") -> None: global _app _app = app + from .models import SnmpDevice # noqa: F401 — registers the table def get_scheduled_tasks() -> list: @@ -32,3 +33,9 @@ def get_alert_metrics() -> dict[str, list[str]]: """SNMP metric names are the OID labels each device is configured with, so the list is empty here and the rule form offers the labels recorded so far.""" return {"snmp": []} + + +def get_audit_entities() -> dict: + """Audit rows about a device name it and link to its page.""" + from .models import SnmpDevice + return {"snmp_device": (SnmpDevice, "/plugins/snmp/devices/{}")} diff --git a/plugins/snmp/migrations/__init__.py b/plugins/snmp/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/snmp/migrations/versions/__init__.py b/plugins/snmp/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/snmp/migrations/versions/snmp_001_devices.py b/plugins/snmp/migrations/versions/snmp_001_devices.py new file mode 100644 index 0000000..aa8906e --- /dev/null +++ b/plugins/snmp/migrations/versions/snmp_001_devices.py @@ -0,0 +1,49 @@ +"""SNMP devices are rows edited on the SNMP page + +Devices lived in plugin.yaml's config, which the UI could not edit, and a poll +that failed left no trace. A row per device holds what to ask and how, and the +newest poll's outcome (milestone 491). + +Revision ID: snmp_001_devices +Revises: (none — a branch off core via depends_on) +Create Date: 2026-10-08 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "snmp_001_devices" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = "snmp" +depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",) + + +def upgrade() -> None: + op.create_table( + "snmp_devices", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("name", sa.String(100), nullable=False, unique=True), + sa.Column("address", sa.String(255), nullable=False), + sa.Column("port", sa.Integer, nullable=False, server_default="161"), + sa.Column("version", sa.String(3), nullable=False, server_default="2c"), + sa.Column("community", sa.Text, nullable=False, server_default=""), + sa.Column("v3_user", sa.String(100), nullable=False, server_default=""), + sa.Column("v3_auth", sa.String(16), nullable=False, server_default="none"), + sa.Column("v3_auth_key", sa.Text, nullable=False, server_default=""), + sa.Column("v3_priv", sa.String(16), nullable=False, server_default="none"), + sa.Column("v3_priv_key", sa.Text, nullable=False, server_default=""), + sa.Column("presets", sa.JSON, nullable=False, server_default="[]"), + sa.Column("interfaces", sa.String(255), nullable=False, server_default=""), + sa.Column("custom", sa.Text, nullable=False, server_default=""), + sa.Column("host_id", sa.String(36), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_polled_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_ok_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_error", sa.Text, nullable=True), + sa.Column("missing", sa.JSON, nullable=False, server_default="[]"), + ) + + +def downgrade() -> None: + op.drop_table("snmp_devices") diff --git a/plugins/snmp/models.py b/plugins/snmp/models.py new file mode 100644 index 0000000..214d6e6 --- /dev/null +++ b/plugins/snmp/models.py @@ -0,0 +1,42 @@ +# plugins/snmp/models.py +from __future__ import annotations + +from datetime import datetime, timezone + +from sqlalchemy import JSON, DateTime, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from steward.core.ids import time_ordered_id +from steward.models.base import Base + + +class SnmpDevice(Base): + """A device Steward polls over SNMP, edited on the SNMP page. + + Secrets (the community, the v3 keys) are stored encrypted. The last_* + columns are the newest poll's outcome, written by the poller, so a page + can say why a device isn't answering without asking the device itself. + """ + __tablename__ = "snmp_devices" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) + name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True) + address: Mapped[str] = mapped_column(String(255), nullable=False) + port: Mapped[int] = mapped_column(Integer, nullable=False, default=161) + version: Mapped[str] = mapped_column(String(3), nullable=False, default="2c") + community: Mapped[str] = mapped_column(Text, nullable=False, default="") + v3_user: Mapped[str] = mapped_column(String(100), nullable=False, default="") + v3_auth: Mapped[str] = mapped_column(String(16), nullable=False, default="none") + v3_auth_key: Mapped[str] = mapped_column(Text, nullable=False, default="") + v3_priv: Mapped[str] = mapped_column(String(16), nullable=False, default="none") + v3_priv_key: Mapped[str] = mapped_column(Text, nullable=False, default="") + presets: Mapped[list] = mapped_column(JSON, nullable=False, default=list) + interfaces: Mapped[str] = mapped_column(String(255), nullable=False, default="") + custom: Mapped[str] = mapped_column(Text, nullable=False, default="") + host_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)) + last_polled_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_ok_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + missing: Mapped[list] = mapped_column(JSON, nullable=False, default=list) diff --git a/plugins/snmp/plugin.yaml b/plugins/snmp/plugin.yaml index 7ef3683..e9c6bb4 100644 --- a/plugins/snmp/plugin.yaml +++ b/plugins/snmp/plugin.yaml @@ -1,7 +1,7 @@ # plugins/snmp/plugin.yaml name: snmp -version: "1.0.0" -description: "SNMP polling for network devices — switches, routers, printers, UPSes, etc." +version: "2.0.0" +description: "SNMP polling for network devices — switches, routers, printers, UPSes, servers." author: "Steward" license: "MIT" min_app_version: "0.1.0" @@ -12,48 +12,6 @@ tags: - network - monitoring +# Devices are added, tested and edited on the SNMP page (/plugins/snmp/). config: poll_interval_seconds: 60 - # Each device is polled independently. OIDs must resolve to numeric values. - # - # `host` is the SNMP poll target (IP/hostname we query). A device also shows - # up on a Steward Host's detail page when it maps to that host. By default - # that mapping is implicit: `host` is matched against the Host's address or - # name. To bind explicitly instead — e.g. when you poll by IP but the Host is - # recorded by DNS name, or for an SNMP-only switch/PDU/UPS — add ONE of: - # host_id: "" # exact Host.id match - # steward_host: "switch01" # Host name OR address (case-insensitive) - # An explicit binding is exclusive: the device then maps ONLY to that host, - # never implicitly to another by a coincidental address string. - devices: - - name: "core-switch" - host: "192.168.1.1" - port: 161 - community: "public" - version: "2c" # "1", "2c", or "3" - oids: - - oid: "1.3.6.1.2.1.1.3.0" - label: "uptime_centisecs" - - oid: "1.3.6.1.2.1.2.2.1.10.1" - label: "if1_in_octets" - - oid: "1.3.6.1.2.1.2.2.1.16.1" - label: "if1_out_octets" - # Example: network UPS with SNMP management card (RFC 1628 UPS-MIB). - # upsBatteryStatus values: 1=unknown, 2=batteryNormal, 3=batteryLow, 4=batteryDepleted. - # Uncomment and point at your UPS to enable. - # - name: "ups" - # host: "192.168.1.10" - # port: 161 - # community: "public" - # version: "2c" - # oids: - # - oid: "1.3.6.1.2.1.33.1.2.1.0" - # label: "battery_status" - # - oid: "1.3.6.1.2.1.33.1.2.2.0" - # label: "seconds_on_battery" - # - oid: "1.3.6.1.2.1.33.1.2.3.0" - # label: "runtime_minutes_remaining" - # - oid: "1.3.6.1.2.1.33.1.2.4.0" - # label: "battery_charge_pct" - # - oid: "1.3.6.1.2.1.33.1.3.3.1.3.1" - # label: "input_voltage_dV" diff --git a/plugins/snmp/poller.py b/plugins/snmp/poller.py index c84de2c..22599e9 100644 --- a/plugins/snmp/poller.py +++ b/plugins/snmp/poller.py @@ -1,19 +1,53 @@ # plugins/snmp/poller.py -""" -Asynchronous SNMP GET helper. +"""Ask an SNMP device for its readings, and say in words why it didn't answer. -Requires pysnmp-lextudio (the maintained pysnmp fork), bundled into the Docker -image via the `snmp` extra (`pip install .[snmp]`): - pip install 'steward[snmp]' - -If pysnmp is not installed, poll_device() returns an empty dict and logs a -warning — SNMP polling is then simply disabled, nothing else breaks. +pysnmp-lextudio 6.x (what the image installs) and canonical pysnmp 7.x put the +asyncio API in different modules with different names; _api() finds whichever +is installed, so a dependency bump can't silently break polling. Without +pysnmp at all the poll reports that instead of raising. """ from __future__ import annotations + import logging +from dataclasses import dataclass, field +from types import SimpleNamespace logger = logging.getLogger(__name__) +TIMEOUT_SECONDS = 5 +# OIDs per GET. A device's whole list usually fits in one request; this keeps +# a long custom list under the size an agent will answer. +OIDS_PER_REQUEST = 24 + +AUTH_PROTOCOLS = {"none": "usmNoAuthProtocol", "md5": "usmHMACMD5AuthProtocol", + "sha": "usmHMACSHAAuthProtocol", "sha224": "usmHMAC128SHA224AuthProtocol", + "sha256": "usmHMAC192SHA256AuthProtocol", "sha384": "usmHMAC256SHA384AuthProtocol", + "sha512": "usmHMAC384SHA512AuthProtocol"} +PRIV_PROTOCOLS = {"none": "usmNoPrivProtocol", "des": "usmDESPrivProtocol", "3des": "usm3DESEDEPrivProtocol", + "aes128": "usmAesCfb128Protocol", "aes192": "usmAesCfb192Protocol", + "aes256": "usmAesCfb256Protocol"} + +# A varbind value of one of these types means the agent has no such object. +_ABSENT = {"NoSuchObject", "NoSuchInstance", "EndOfMibView"} + + +@dataclass +class Credentials: + version: str = "2c" + community: str = "" + user: str = "" + auth: str = "none" + auth_key: str = "" + priv: str = "none" + priv_key: str = "" + + +@dataclass +class PollResult: + readings: dict[str, float] = field(default_factory=dict) + error: str | None = None # the device didn't answer, and why + missing: list[str] = field(default_factory=list) # keys it answered without + def _pysnmp_available() -> bool: try: @@ -23,9 +57,27 @@ def _pysnmp_available() -> bool: return False -def _mp_model(version: str) -> int: - """Map an SNMP version string to pysnmp's mpModel int (0 = v1, 1 = v2c).""" - return 0 if version == "1" else 1 +def _api() -> SimpleNamespace: + """The installed pysnmp's asyncio API under one set of names.""" + try: + from pysnmp.hlapi.v3arch import asyncio as hl + from pysnmp.hlapi.v3arch.asyncio import auth + return SimpleNamespace(hl=hl, auth=auth, get=hl.get_cmd, walk=hl.walk_cmd, async_transport=True) + except ImportError: + from pysnmp.hlapi import asyncio as hl + from pysnmp.hlapi.asyncio import auth + return SimpleNamespace(hl=hl, auth=auth, get=hl.getCmd, walk=hl.walkCmd, async_transport=False) + + +def _auth_data(api, creds: Credentials): + if creds.version == "3": + return api.hl.UsmUserData( + creds.user, + authKey=creds.auth_key or None, + privKey=creds.priv_key or None, + authProtocol=getattr(api.auth, AUTH_PROTOCOLS.get(creds.auth, "usmNoAuthProtocol")), + privProtocol=getattr(api.auth, PRIV_PROTOCOLS.get(creds.priv, "usmNoPrivProtocol"))) + return api.hl.CommunityData(creds.community, mpModel=0 if creds.version == "1" else 1) def _close_engine(engine) -> None: @@ -66,106 +118,149 @@ def _close_engine(engine) -> None: return -async def poll_device( - host: str, - port: int, - community: str, - version: str, - oids: list[dict], -) -> dict[str, float]: - """Perform an SNMP GET for each OID and return ``{label: float_value}``. +def explain(error_indication, address: str, port: int, creds: Credentials) -> str: + """pysnmp's error indication, said as what to check.""" + text = str(error_indication) + low = text.lower() + if "timeout" in low or "no snmp response" in low: + who = (f"the v3 user “{creds.user}”" if creds.version == "3" + else "the community") + return (f"No reply from {address}:{port} within {TIMEOUT_SECONDS} s. Check the address and port, " + f"that SNMP is on, and {who}: a device ignores a request it doesn't accept.") + if "unknown" in low and "user" in low: + return f"The device doesn't know the v3 user “{creds.user}”." + if "digest" in low or "authentication" in low: + return "The device refused the v3 authentication: check the auth protocol and key." + if "decrypt" in low or "privacy" in low: + return "The device couldn't decrypt the request: check the privacy protocol and key." + if "not in time window" in low: + return "The device rejected the request as out of its time window; try again." + return f"SNMP error from {address}:{port}: {text}" - Non-numeric OIDs (strings, etc.) are skipped. Returns an empty dict on any - error (unreachable host, wrong community, …) so a flaky device never breaks - the poll loop. - pysnmp's HLAPI is asyncio-only as of v6. The import path moved between major - versions, so we support both rather than let a dependency bump silently - re-break polling: - • pysnmp-lextudio 6.2.x → ``pysnmp.hlapi.asyncio`` (``getCmd`` + a directly - constructed ``UdpTransportTarget``). - • canonical pysnmp 7.x → ``pysnmp.hlapi.v3arch.asyncio`` (``get_cmd`` + the - async ``UdpTransportTarget.create``). +async def _transport(api, address: str, port: int): + if api.async_transport: + return await api.hl.UdpTransportTarget.create((address, port), timeout=TIMEOUT_SECONDS, retries=1) + return api.hl.UdpTransportTarget((address, port), timeout=TIMEOUT_SECONDS, retries=1) + + +async def poll_device(address: str, port: int, creds: Credentials, + oids: list[tuple[str, str]]) -> PollResult: + """GET every (oid, key) of a device, a few dozen to a request. + + Returns the numeric answers by key, the keys the device answered without + (no such object, or not a number), and an error in words when it didn't + answer at all. Never raises: a broken device must not stop the others. """ + result = PollResult() + if not oids: + return result if not _pysnmp_available(): - logger.warning("pysnmp not installed — SNMP polling disabled. " - "Install with: pip install 'steward[snmp]'") - return {} + result.error = "pysnmp isn't installed in this image, so SNMP polling is off." + return result + api = _api() + engine = api.hl.SnmpEngine() try: - # canonical pysnmp 7.x - from pysnmp.hlapi.v3arch.asyncio import ( - CommunityData, - ContextData, - ObjectIdentity, - ObjectType, - SnmpEngine, - UdpTransportTarget, - get_cmd as _get_cmd, - ) - _transport_is_async = True - except ImportError: - # pysnmp-lextudio 6.2.x - from pysnmp.hlapi.asyncio import ( - CommunityData, - ContextData, - ObjectIdentity, - ObjectType, - SnmpEngine, - UdpTransportTarget, - getCmd as _get_cmd, - ) - _transport_is_async = False - - engine = SnmpEngine() - - # Always release the engine's UDP socket — see _close_engine for why. - try: - # Same host/port for every OID on this device, so build the transport once. try: - if _transport_is_async: - transport = await UdpTransportTarget.create((host, port), timeout=5, retries=1) - else: - transport = UdpTransportTarget((host, port), timeout=5, retries=1) + transport = await _transport(api, address, port) except Exception as exc: - logger.debug("SNMP transport setup failed for %s:%s: %s", host, port, exc) - return {} + result.error = f"Can't reach {address}:{port}: {exc}" + return result + auth = _auth_data(api, creds) - results: dict[str, float] = {} - - for oid_cfg in oids: - oid = oid_cfg["oid"] - label = oid_cfg.get("label") or oid - scale = float(oid_cfg.get("scale", 1.0)) - - try: - error_indication, error_status, error_index, var_binds = await _get_cmd( - engine, - CommunityData(community, mpModel=_mp_model(version)), - transport, - ContextData(), - ObjectType(ObjectIdentity(oid)), - ) - except Exception as exc: - logger.debug("SNMP GET %s@%s OID %s failed: %s", host, port, oid, exc) - continue - - if error_indication: - logger.debug("SNMP error %s@%s OID %s: %s", host, port, oid, error_indication) - continue - if error_status: - logger.debug("SNMP status %s@%s OID %s: %s at %s", - host, port, oid, error_status.prettyPrint(), - error_index and var_binds[int(error_index) - 1][0] or "?") - continue - - for _, val in var_binds: + pending = list(oids) + while pending: + chunk, pending = pending[:OIDS_PER_REQUEST], pending[OIDS_PER_REQUEST:] + ei, es, eidx, var_binds = await api.get( + engine, auth, transport, api.hl.ContextData(), + *[api.hl.ObjectType(api.hl.ObjectIdentity(oid)) for oid, _ in chunk], + lookupMib=False) + if ei: + result.error = explain(ei, address, port, creds) + return result + if es: + # v1 fails the whole request on the first unknown OID: note it, + # and ask again for the rest. + bad = int(eidx) - 1 if eidx else -1 + if 0 <= bad < len(chunk): + result.missing.append(chunk[bad][1]) + pending = chunk[:bad] + chunk[bad + 1:] + pending + continue + result.error = f"The device answered with an error: {es.prettyPrint()}" + return result + # A GET answers in the order it asked. + for (_, key), (_, val) in zip(chunk, var_binds): + if type(val).__name__ in _ABSENT: + result.missing.append(key) + continue try: - results[label] = float(val) * scale + result.readings[key] = float(val) except (TypeError, ValueError): - # Non-numeric type (e.g. OctetString description) — skip. - logger.debug("SNMP non-numeric value for %s label=%s: %r", oid, label, val) + result.missing.append(key) + if not result.readings and result.missing: + result.error = "The device answered but has none of these readings." + return result + except Exception as exc: + logger.warning("SNMP poll of %s:%s failed: %s", address, port, exc) + result.error = f"SNMP poll failed: {exc}" + return result + finally: + _close_engine(engine) - return results + +SYS_DESCR, SYS_UPTIME, SYS_NAME = "1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0" +IF_DESCR, IF_NAME = "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.31.1.1.1.1" +INTERFACES_LISTED = 128 + + +async def describe_device(address: str, port: int, creds: Credentials) -> dict: + """What Test shows: {"error"} or {"name", "description", "uptime_cs", + "interfaces": [(number, name)]}. Asks the device directly, so only an + operator's explicit Test calls it, never a page load.""" + if not _pysnmp_available(): + return {"error": "pysnmp isn't installed in this image, so SNMP is off."} + api = _api() + engine = api.hl.SnmpEngine() + try: + try: + transport = await _transport(api, address, port) + except Exception as exc: + return {"error": f"Can't reach {address}:{port}: {exc}"} + auth = _auth_data(api, creds) + ei, es, _, var_binds = await api.get( + engine, auth, transport, api.hl.ContextData(), + *[api.hl.ObjectType(api.hl.ObjectIdentity(o)) for o in (SYS_NAME, SYS_DESCR, SYS_UPTIME)], + lookupMib=False) + if ei: + return {"error": explain(ei, address, port, creds)} + if es: + return {"error": f"The device answered with an error: {es.prettyPrint()}"} + values = [None if type(v).__name__ in _ABSENT else v for _, v in var_binds] + out = {"name": str(values[0]) if values[0] is not None else "", + "description": str(values[1]) if values[1] is not None else "", + "uptime_cs": float(values[2]) if values[2] is not None else None, + "interfaces": []} + # ifName where the device has it (IF-MIB), else ifDescr. + for column in (IF_NAME, IF_DESCR): + found: list[tuple[int, str]] = [] + async for ei, es, _, rows in api.walk(engine, auth, transport, api.hl.ContextData(), + api.hl.ObjectType(api.hl.ObjectIdentity(column)), + lexicographicMode=False, lookupMib=False): + if ei or es: + break + for name, val in rows: + if type(val).__name__ in _ABSENT: + continue + found.append((int(str(name).rsplit(".", 1)[-1]), str(val))) + if len(found) >= INTERFACES_LISTED: + break + if found: + out["interfaces"] = found + break + return out + except Exception as exc: + logger.warning("SNMP test of %s:%s failed: %s", address, port, exc) + return {"error": f"SNMP test failed: {exc}"} finally: _close_engine(engine) diff --git a/plugins/snmp/readings.py b/plugins/snmp/readings.py new file mode 100644 index 0000000..514d8e8 --- /dev/null +++ b/plugins/snmp/readings.py @@ -0,0 +1,213 @@ +# plugins/snmp/readings.py +"""What an SNMP device is asked for, and how each answer reads. + +A device's readings come from the presets ticked on its form plus any custom +lines. Each reading has a key (the metric name it is stored and alerted on), +a name a person reads, and a unit. A counter (an interface's byte count, any +"total so far") is stored as a rate: the poller turns two samples into a +per-second figure, so a chart and an alert see traffic, not a lifetime total. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass + +from steward.core.formatting import human_bytes, human_duration + + +@dataclass(frozen=True) +class Reading: + oid: str + key: str + name: str + unit: str = "number" + scale: float = 1.0 + offset: float = 0.0 + + @property + def counter(self) -> bool: + return self.unit in COUNTER_UNITS + + +# A counter's unit names what it counts; it is stored as the rate unit it maps +# to, multiplied by the factor (a byte counter is stored as bits per second). +COUNTER_UNITS = {"octets": ("bits_per_second", 8.0), "count": ("per_second", 1.0)} + +STATUS_WORDS = { + "if_status": {1: "up", 2: "down", 3: "testing", 4: "unknown", 5: "dormant", + 6: "not present", 7: "lower layer down"}, + "ups_battery": {1: "unknown", 2: "normal", 3: "low", 4: "depleted"}, +} + +# The units a custom line may name, with how each one reads, for the form's hint. +UNITS = { + "number": "a plain number", + "percent": "0 to 100", + "celsius": "degrees Celsius", + "volts": "volts", + "decivolts": "tenths of a volt", + "seconds": "a span in seconds", + "minutes": "a span in minutes", + "centiseconds": "a span in hundredths of a second (sysUpTime)", + "bytes": "a size in bytes", + "kilobytes": "a size in KiB", + "octets": "a byte counter, shown as traffic (bit/s)", + "count": "any other counter, shown per second", +} + +_UPTIME = Reading("1.3.6.1.2.1.1.3.0", "uptime", "Uptime", "centiseconds") + +# id: (label, what it is for, readings). Interface traffic is per interface +# and built by interface_readings from the numbers on the form. +PRESETS: dict[str, tuple[str, str, tuple[Reading, ...]]] = { + "system": ("System", "How long the device has been up. Nearly every SNMP device answers it.", + (_UPTIME,)), + "ups": ("UPS", "Battery state, charge, runtime left, load and input voltage (UPS-MIB, RFC 1628; " + "APC, Eaton and CyberPower management cards).", + (Reading("1.3.6.1.2.1.33.1.2.1.0", "battery_status", "Battery", "ups_battery"), + Reading("1.3.6.1.2.1.33.1.2.4.0", "battery_charge", "Charge", "percent"), + Reading("1.3.6.1.2.1.33.1.2.3.0", "runtime_left", "Runtime left", "minutes"), + Reading("1.3.6.1.2.1.33.1.4.4.1.5.1", "load", "Load", "percent"), + Reading("1.3.6.1.2.1.33.1.3.3.1.3.1", "input_voltage", "Input", "volts"), + Reading("1.3.6.1.2.1.33.1.2.2.0", "on_battery_for", "On battery for", "seconds"))), + "linux": ("Linux server", "CPU, memory and load from net-snmp (snmpd's UCD-SNMP-MIB).", + (Reading("1.3.6.1.4.1.2021.11.11.0", "cpu_used", "CPU used", "percent", scale=-1.0, offset=100.0), + Reading("1.3.6.1.4.1.2021.10.1.5.1", "load_1m", "Load (1m)", "number", scale=0.01), + Reading("1.3.6.1.4.1.2021.4.6.0", "memory_free", "Memory free", "kilobytes"), + Reading("1.3.6.1.4.1.2021.4.5.0", "memory_total", "Memory total", "kilobytes"))), +} + +PRESET_ORDER = ("system", "interfaces", "ups", "linux") +INTERFACES_LABEL = ("Interface traffic", "Traffic in and out and link state for the interfaces numbered " + "below. Test the device to see its interfaces and their numbers.") + + +def parse_interfaces(text: str) -> list[int]: + """'1, 2, 5-7' → [1, 2, 5, 6, 7]. Raises ValueError naming what it can't read.""" + out: list[int] = [] + for part in re.split(r"[,\s]+", (text or "").strip()): + if not part: + continue + m = re.fullmatch(r"(\d+)(?:-(\d+))?", part) + if not m: + raise ValueError(f"“{part}” is not an interface number or range") + lo, hi = int(m.group(1)), int(m.group(2) or m.group(1)) + if hi < lo or hi - lo > 64: + raise ValueError(f"“{part}” is not a range of up to 64 interfaces") + out.extend(n for n in range(lo, hi + 1) if n not in out) + return out + + +def interface_readings(numbers: list[int], version: str) -> list[Reading]: + """In, out and link state per interface. v2c and v3 read the 64-bit + counters (ifHCInOctets), which don't wrap on a fast link; v1 has only the + 32-bit ones (ifInOctets).""" + hc = version != "1" + out: list[Reading] = [] + for n in numbers: + rx = f"1.3.6.1.2.1.31.1.1.1.6.{n}" if hc else f"1.3.6.1.2.1.2.2.1.10.{n}" + tx = f"1.3.6.1.2.1.31.1.1.1.10.{n}" if hc else f"1.3.6.1.2.1.2.2.1.16.{n}" + out += [Reading(f"1.3.6.1.2.1.2.2.1.8.{n}", f"if{n}_status", f"Interface {n}", "if_status"), + Reading(rx, f"if{n}_in", f"Interface {n} in", "octets"), + Reading(tx, f"if{n}_out", f"Interface {n} out", "octets")] + return out + + +_OID = re.compile(r"\.?\d+(\.\d+)+") +_KEY = re.compile(r"[a-z][a-z0-9_]{0,63}") + + +def parse_custom(text: str) -> list[Reading]: + """One reading a line: `OID name [unit]`. The name is a single word + (letters, digits, underscores) and becomes the metric name; the unit is + one of UNITS and defaults to number. Raises ValueError naming the line.""" + out: list[Reading] = [] + for i, raw in enumerate((text or "").splitlines(), start=1): + line = raw.split("#", 1)[0].strip() + if not line: + continue + parts = line.split() + if len(parts) not in (2, 3): + raise ValueError(f"Line {i}: write it as “OID name” or “OID name unit”") + oid, key = parts[0].lstrip("."), parts[1].lower() + unit = parts[2].lower() if len(parts) == 3 else "number" + if not _OID.fullmatch(oid): + raise ValueError(f"Line {i}: “{parts[0]}” is not an OID (numbers separated by dots)") + if not _KEY.fullmatch(key): + raise ValueError(f"Line {i}: the name “{parts[1]}” must start with a letter and use " + "only letters, digits and underscores") + if unit not in UNITS: + raise ValueError(f"Line {i}: “{parts[2]}” is not a unit; use one of {', '.join(UNITS)}") + out.append(Reading(oid, key, key.replace("_", " ").capitalize(), unit)) + return out + + +def device_readings(presets: list[str], interfaces: list[int], custom: str, version: str) -> list[Reading]: + """Every reading a device is asked for, presets in their order then custom + lines. Raises ValueError when two readings share a name.""" + out: list[Reading] = [] + for pid in PRESET_ORDER: + if pid == "interfaces": + if "interfaces" in presets: + out += interface_readings(interfaces, version) + elif pid in presets: + out += PRESETS[pid][2] + out += parse_custom(custom) + seen: set[str] = set() + for r in out: + if r.key in seen: + raise ValueError(f"Two readings are named “{r.key}”; give the custom one another name") + seen.add(r.key) + return out + + +def stored_unit(reading: Reading) -> str: + """The unit a reading is stored in: a counter's rate unit, else its own.""" + return COUNTER_UNITS[reading.unit][0] if reading.counter else reading.unit + + +def _bits(bps: float) -> str: + for unit, size in (("Gbit/s", 1e9), ("Mbit/s", 1e6), ("kbit/s", 1e3)): + if abs(bps) >= size: + return f"{bps / size:.3g} {unit}" + return f"{bps:.0f} bit/s" + + +def format_value(unit: str, value: float | None) -> str: + """A stored value as a person reads it, in its unit.""" + if value is None: + return "—" + if unit in STATUS_WORDS: + return STATUS_WORDS[unit].get(int(value), f"code {value:.0f}") + if unit == "centiseconds": + return human_duration(value / 100) + if unit == "seconds": + return human_duration(value) + if unit == "minutes": + return human_duration(value * 60) + if unit == "percent": + return f"{value:.0f}%" + if unit == "celsius": + return f"{value:.1f} °C" + if unit == "volts": + return f"{value:.0f} V" + if unit == "decivolts": + return f"{value / 10:.1f} V" + if unit == "bytes": + return human_bytes(value) + if unit == "kilobytes": + return human_bytes(value * 1024) + if unit == "bits_per_second": + return _bits(value) + if unit == "per_second": + return f"{value:.3g}/s" + return f"{value:,.0f}" if float(value).is_integer() else f"{value:.4g}" + + +# What a chart's axis says for a stored unit, and how its values scale for it. +def chart_axis(unit: str) -> tuple[str, float]: + return {"centiseconds": ("days", 1 / 8_640_000), "seconds": ("seconds", 1.0), + "minutes": ("minutes", 1.0), "percent": ("%", 1.0), "celsius": ("°C", 1.0), + "volts": ("V", 1.0), "decivolts": ("V", 0.1), "bytes": ("MiB", 1 / 1_048_576), + "kilobytes": ("MiB", 1 / 1024), "bits_per_second": ("Mbit/s", 1e-6), + "per_second": ("per second", 1.0)}.get(unit, ("", 1.0)) diff --git a/plugins/snmp/routes.py b/plugins/snmp/routes.py index cec73e6..8b223a8 100644 --- a/plugins/snmp/routes.py +++ b/plugins/snmp/routes.py @@ -1,251 +1,319 @@ # plugins/snmp/routes.py +"""The SNMP pages. Every GET reads Steward's own records: the device rows the +editor writes and the poller annotates, and the readings it stores. Only Test, +an admin's explicit action, asks a device anything (System 178's ruling).""" from __future__ import annotations + +import json from datetime import datetime, timedelta, timezone -from quart import Blueprint, current_app, render_template, request -from sqlalchemy import select +from quart import Blueprint, abort, current_app, redirect, render_template, request, session +from sqlalchemy import delete, select, update from steward.auth.middleware import require_role -from steward.models.users import UserRole -from steward.core.formatting import human_bytes, human_duration +from steward.core.audit import log_audit from steward.core.metric_series import latest_by_resource -from steward.models.metrics import PluginMetric +from steward.models.metrics import PluginMetric, PluginMetricLatest +from steward.models.users import UserRole + +from .models import SnmpDevice +from .readings import (INTERFACES_LABEL, PRESETS, PRESET_ORDER, UNITS, chart_axis, format_value, + stored_unit) +from .service import (FormError, device_state, form_credentials, form_values, parse_form, + readings_for) snmp_bp = Blueprint("snmp", __name__, template_folder="templates") -async def _latest_readings(db, device_names: list[str]) -> dict[str, dict]: - """{device_name: {"readings": {label: latest_value}, "at": newest reading's - time}} for the devices that have any readings.""" - latest = await latest_by_resource(db, "snmp", device_names) - return {name: {"readings": {m: r.value for m, r in metrics.items()}, - "at": max(r.recorded_at for r in metrics.values())} - for name, metrics in latest.items() if metrics} - - -# A device whose newest reading is older than this many poll intervals is not -# shown as reachable: the poller has stopped getting answers from it. -STALE_POLLS = 3 - - -def _device_state(entry: dict | None, poll_seconds: int, now: datetime) -> dict: - """What the page says about a device's readings: {"readings", "at", - "reach"}, reach being "up" (answered within STALE_POLLS polls), "stale" - (has readings, but none recent) or "pending" (never answered).""" - if not entry: - return {"readings": {}, "at": None, "reach": "pending"} - fresh = (now - entry["at"]).total_seconds() <= STALE_POLLS * max(int(poll_seconds), 1) - return {**entry, "reach": "up" if fresh else "stale"} - - def _poll_seconds() -> int: return int(current_app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60)) -async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]: - """Return {label: [(recorded_at, value), ...]} for a single device.""" - since = datetime.now(timezone.utc) - timedelta(hours=hours) - result = await db.execute( - select(PluginMetric) - .where( - PluginMetric.source_module == "snmp", - PluginMetric.resource_name == device_name, - PluginMetric.recorded_at >= since, - ) - .order_by(PluginMetric.recorded_at) - ) - rows = result.scalars().all() +async def _all_devices(db) -> list[SnmpDevice]: + return list((await db.execute(select(SnmpDevice).order_by(SnmpDevice.name))).scalars().all()) - out: dict[str, list] = {} - for row in rows: - out.setdefault(row.metric_name, []).append( - (row.recorded_at.isoformat(), row.value) - ) - return out + +async def _latest(db, names: list[str]) -> dict[str, dict]: + """{device name: {key: value}} from plugin_metric_latest.""" + latest = await latest_by_resource(db, "snmp", names) + return {name: {m: r.value for m, r in metrics.items()} for name, metrics in latest.items()} + + +def shown_readings(device: SnmpDevice, values: dict) -> list[dict]: + """The device's readings in its own order, each named and in its unit; + "—" for one with no value yet.""" + return [{"key": r.key, "name": r.name, "counter": r.counter, + "text": format_value(stored_unit(r), values.get(r.key))} + for r in readings_for(device)] + + +def view(device: SnmpDevice, values: dict, now: datetime) -> dict: + state = device_state(device, _poll_seconds(), now) + names = {r.key: r.name for r in readings_for(device)} + return {"id": device.id, "name": device.name, "address": device.address, "port": device.port, + "version": device.version, "reach": state.reach, "says": state.says, + "polled_at": device.last_polled_at, "ok_at": device.last_ok_at, + "readings": shown_readings(device, values), + "missing": [names.get(k, k) for k in device.missing or []]} + + +_REACH_ORDER = {"down": 0, "stale": 1, "pending": 2, "up": 3} @snmp_bp.get("/") @require_role(UserRole.viewer) async def index(): - devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) - device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg] - async with current_app.db_sessionmaker() as db: - latest = await _latest_readings(db, device_names) + devices = await _all_devices(db) + latest = await _latest(db, [d.name for d in devices]) now = datetime.now(timezone.utc) - - # Merge config + latest readings for the template - devices = [] - for cfg in devices_cfg: - name = cfg.get("name") or cfg.get("host", "?") - devices.append({ - "name": name, - "host": cfg.get("host", ""), - "oids": cfg.get("oids", []), - **_device_state(latest.get(name), _poll_seconds(), now), - }) - - poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) - return await render_template("snmp/index.html", - devices=devices, - poll_interval=poll_interval) + shown = [view(d, latest.get(d.name, {}), now) for d in devices] + shown.sort(key=lambda v: (_REACH_ORDER[v["reach"]], v["name"].lower())) + return await render_template("snmp/index.html", devices=shown) -def _device_binds_to_host(d: dict, keys: set[str], host_id: str) -> bool: - """Does config device `d` belong on the host identified by `keys` - (lowercased {address, name}) / `host_id`? +# ── The editor ──────────────────────────────────────────────────────────────── - A device may declare an **explicit** binding that decouples "which Steward - host this belongs to" from "where to poll" (the `host` field): - • `host_id` — the Steward Host UUID (exact match), the explicit link. - • `steward_host` — a friendly bind by Host name or address (case-insensitive). - An explicit binding is **exclusive**: a device bound to host A must not also - match host B by a coincidental poll-target string. Only when no explicit - binding is present do we fall back to the implicit `host`-string match - (backward compatible with configs that only set `host`). - """ - explicit_id = str(d.get("host_id", "")).strip() - explicit_name = str(d.get("steward_host", "")).strip().lower() - if explicit_id or explicit_name: - if explicit_id and host_id and explicit_id == host_id: - return True - return bool(explicit_name and explicit_name in keys) - return str(d.get("host", "")).strip().lower() in keys - - -def _devices_for_host(devices_cfg: list, address: str | None, name: str | None, - host_id: str | None = None) -> list[dict]: - """SNMP devices that map onto a Steward host's page. Prefers an explicit - per-device binding (`host_id` / `steward_host`); otherwise falls back to - matching the device's `host` (poll target) against the host's address or - name (case-insensitive). See `_device_binds_to_host` for the precedence.""" - keys = {(address or "").strip().lower(), (name or "").strip().lower()} - keys.discard("") - hid = (host_id or "").strip() - return [ - d for d in devices_cfg - if isinstance(d, dict) and _device_binds_to_host(d, keys, hid) - ] - - -@snmp_bp.get("/host/") -@require_role(UserRole.viewer) -async def host_panel(host_id: str): - """Per-host SNMP fragment for the Hosts hub. Surfaces any configured SNMP - device that maps to this host (by address or name) with its latest readings. - Renders nothing when no device maps, so hosts without SNMP carry no empty card. - """ +async def _hosts(db) -> list: from steward.models.hosts import Host + return list((await db.execute(select(Host.id, Host.name).order_by(Host.name))).all()) - devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) + +async def _render_form(device: SnmpDevice | None, values: dict, error: str | None = None, status: int = 200): async with current_app.db_sessionmaker() as db: - host = (await db.execute( - select(Host).where(Host.id == host_id))).scalar_one_or_none() - if host is None: - return "" - matched = _devices_for_host(devices_cfg, host.address, host.name, host.id) - if not matched: - return "" - names = [d.get("name") or d.get("host", "?") for d in matched] - latest = await _latest_readings(db, names) + hosts = await _hosts(db) + presets = [(pid, *(INTERFACES_LABEL if pid == "interfaces" else PRESETS[pid][:2])) for pid in PRESET_ORDER] + return await render_template("snmp/form.html", device=device, v=values, error=error, hosts=hosts, + presets=presets, units=UNITS), status + + +def _actor() -> tuple[str | None, str]: + return session.get("user_id"), session.get("username", "") + + +@snmp_bp.get("/devices/new") +@require_role(UserRole.admin) +async def new_device(): + return await _render_form(None, form_values(None)) + + +@snmp_bp.post("/devices/new") +@require_role(UserRole.admin) +async def create_device(): + form = await request.form + try: + values = parse_form(form) + except FormError as exc: + return await _render_form(None, _typed(form), str(exc), 422) + async with current_app.db_sessionmaker() as db: + async with db.begin(): + if await _name_taken(db, values["name"]): + return await _render_form(None, _typed(form), f"A device is already named “{values['name']}”.", 422) + device = SnmpDevice(**values) + db.add(device) + await db.flush() + device_id = device.id + await log_audit(current_app, *_actor(), "snmp.device_created", entity_type="snmp_device", + entity_id=device_id, detail={"name": values["name"], "address": values["address"]}) + return redirect(f"/plugins/snmp/devices/{device_id}") + + +async def _name_taken(db, name: str, other_than: str | None = None) -> bool: + q = select(SnmpDevice.id).where(SnmpDevice.name == name) + if other_than: + q = q.where(SnmpDevice.id != other_than) + return (await db.execute(q)).first() is not None + + +def _typed(form) -> dict: + """What was typed, to show the form again with an error. Secrets are not + echoed back.""" + return {"name": form.get("name", ""), "address": form.get("address", ""), "port": form.get("port", ""), + "version": form.get("version", "2c"), "v3_user": form.get("v3_user", ""), + "v3_auth": form.get("v3_auth", "none"), "v3_priv": form.get("v3_priv", "none"), + "presets": form.getlist("presets"), "interfaces": form.get("interfaces", ""), + "custom": form.get("custom", ""), "host_id": form.get("host_id", "")} + + +async def _device_or_404(db, device_id: str) -> SnmpDevice: + device = await db.get(SnmpDevice, device_id) + if device is None: + abort(404) + return device + + +@snmp_bp.get("/devices//edit") +@require_role(UserRole.admin) +async def edit_device(device_id: str): + async with current_app.db_sessionmaker() as db: + device = await _device_or_404(db, device_id) + return await _render_form(device, form_values(device)) + + +@snmp_bp.post("/devices//edit") +@require_role(UserRole.admin) +async def save_device(device_id: str): + form = await request.form + async with current_app.db_sessionmaker() as db: + async with db.begin(): + device = await _device_or_404(db, device_id) + try: + values = parse_form(form, device) + except FormError as exc: + return await _render_form(device, {**_typed(form), **_kept_secrets(device)}, str(exc), 422) + if await _name_taken(db, values["name"], other_than=device.id): + return await _render_form(device, {**_typed(form), **_kept_secrets(device)}, + f"A device is already named “{values['name']}”.", 422) + old_name = device.name + for column, value in values.items(): + setattr(device, column, value) + # What it is asked, or how, may have changed: the last outcome no + # longer describes it, so the page says "not polled yet" until the next poll. + device.last_polled_at = device.last_ok_at = device.last_error = None + device.missing = [] + if old_name != device.name: + # Its history is stored under its name; move it with the rename. + for model in (PluginMetric, PluginMetricLatest): + await db.execute(update(model).where( + model.source_module == "snmp", model.resource_name == old_name + ).values(resource_name=device.name)) + from . import scheduler + if scheduler.RATES is not None: + scheduler.RATES.forget(device_id) + detail = {"name": values["name"]} + if old_name != values["name"]: + detail["old_name"] = old_name + await log_audit(current_app, *_actor(), "snmp.device_updated", entity_type="snmp_device", + entity_id=device_id, detail=detail) + return redirect(f"/plugins/snmp/devices/{device_id}") + + +def _kept_secrets(device: SnmpDevice) -> dict: + return {"has_community": bool(device.community), "has_auth_key": bool(device.v3_auth_key), + "has_priv_key": bool(device.v3_priv_key)} + + +@snmp_bp.post("/devices//delete") +@require_role(UserRole.admin) +async def delete_device(device_id: str): + async with current_app.db_sessionmaker() as db: + async with db.begin(): + device = await _device_or_404(db, device_id) + name = device.name + await db.delete(device) + # Its newest readings go with it, so nothing reads it as a device + # that stopped answering. Its history ages out with retention. + await db.execute(delete(PluginMetricLatest).where( + PluginMetricLatest.source_module == "snmp", PluginMetricLatest.resource_name == name)) + await log_audit(current_app, *_actor(), "snmp.device_deleted", entity_type="snmp_device", + entity_id=device_id, detail={"name": name}) + return redirect("/plugins/snmp/") + + +@snmp_bp.post("/devices/test") +@require_role(UserRole.admin) +async def test_device(): + """Ask the device on the (unsaved) form who it is and what interfaces it + has. An admin's explicit action, so it may contact the device.""" + from .poller import describe_device + form = await request.form + existing = None + if form.get("device_id"): + async with current_app.db_sessionmaker() as db: + existing = await db.get(SnmpDevice, form["device_id"]) + address, port, creds = form_credentials(form, existing) + if not address: + found = {"error": "Enter the device's address first."} + else: + found = await describe_device(address, port, creds) + return await render_template("snmp/_test.html", found=found, address=address, port=port) + + +# ── The device page, the dashboard widget, the host page's panel ───────────── + +async def _history(db, device_name: str, since: datetime) -> dict[str, list]: + """{key: [(iso time, value), ...]} for one device since `since`.""" + rows = (await db.execute( + select(PluginMetric.metric_name, PluginMetric.recorded_at, PluginMetric.value) + .where(PluginMetric.source_module == "snmp", PluginMetric.resource_name == device_name, + PluginMetric.recorded_at >= since) + .order_by(PluginMetric.recorded_at))).all() + out: dict[str, list] = {} + for key, at, value in rows: + out.setdefault(key, []).append((at.isoformat(), value)) + return out + + +RANGES = (1, 6, 24, 168) + + +@snmp_bp.get("/devices/") +@require_role(UserRole.viewer) +async def device_detail(device_id: str): + hours = request.args.get("hours", type=int, default=24) + if hours not in RANGES: + hours = 24 now = datetime.now(timezone.utc) - - devices = [{ - "name": d.get("name") or d.get("host", "?"), - "host": d.get("host", ""), - "oids": d.get("oids", []), - **_device_state(latest.get(d.get("name") or d.get("host", "?")), _poll_seconds(), now), - "bound": bool(str(d.get("host_id", "")).strip() - or str(d.get("steward_host", "")).strip()), - } for d in matched] - return await render_template("snmp/host_panel.html", devices=devices) + async with current_app.db_sessionmaker() as db: + device = await _device_or_404(db, device_id) + latest = await _latest(db, [device.name]) + hist = await _history(db, device.name, now - timedelta(hours=hours)) + charts = [] + for r in readings_for(device): + unit = stored_unit(r) + axis, factor = chart_axis(unit) + charts.append({"key": r.key, "name": r.name, "axis": axis, + "now": format_value(unit, latest.get(device.name, {}).get(r.key)), + "points": [(t, v * factor) for t, v in hist.get(r.key, [])]}) + return await render_template("snmp/device.html", device=view(device, latest.get(device.name, {}), now), + charts=charts, charts_json=json.dumps(charts), hours=hours, ranges=RANGES) -# Labels by suffix: how a reading reads ("uptime_centisecs" → uptime 12d 4h). -# Counters (octets, packets) only ever grow, so the raw number says little; -# a device's key readings prefer the others. -_COUNTER_SUFFIXES = ("_octets", "_packets", "_pkts", "_errors", "_discards") - - -def snmp_reading(label: str, value: float) -> tuple[str, str]: - """(name, value) as a reader would say them: units taken from the label's - suffix, underscores as spaces.""" - if label.endswith("_centisecs"): - return label[:-len("_centisecs")].replace("_", " "), human_duration(value / 100) - if label.endswith("_pct"): - return label[:-len("_pct")].replace("_", " "), f"{value:.0f}%" - if label.endswith("_dV"): - return label[:-len("_dV")].replace("_", " "), f"{value / 10:.1f} V" - if label.endswith("_octets"): - return label[:-len("_octets")].replace("_", " "), f"{human_bytes(value)} total" - return label.replace("_", " "), f"{value:,.0f}" if float(value).is_integer() else f"{value:.4g}" - - -def key_readings(oids: list[dict], readings: dict, limit: int = 2) -> list[tuple[str, str]]: - """The first `limit` readings in config order, gauges before counters.""" - labels = [o.get("label") or o.get("oid") for o in oids if isinstance(o, dict)] - present = [lb for lb in labels if lb in readings] - ordered = ([lb for lb in present if not lb.endswith(_COUNTER_SUFFIXES)] - + [lb for lb in present if lb.endswith(_COUNTER_SUFFIXES)]) - return [snmp_reading(lb, readings[lb]) for lb in ordered[:limit]] - - -_REACH_ORDER = {"stale": 0, "pending": 1, "up": 2} DEVICES_SHOWN = 10 @snmp_bp.get("/widget") @require_role(UserRole.viewer) async def widget(): - """The dashboard SNMP widget (#5255), in the widget-shape order: a - verdict, devices not answering first with since when, then the rest, each - with its status in words and one or two readable readings.""" - devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) - device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg] - + """The dashboard SNMP widget (#5255), in widget shape: a verdict, devices + not answering first with why, then the rest with two readings each.""" async with current_app.db_sessionmaker() as db: - latest = await _latest_readings(db, device_names) + devices = await _all_devices(db) + latest = await _latest(db, [d.name for d in devices]) now = datetime.now(timezone.utc) - - devices = [] - for cfg in devices_cfg: - name = cfg.get("name") or cfg.get("host", "?") - state = _device_state(latest.get(name), _poll_seconds(), now) - devices.append({"name": name, **state, - "key": key_readings(cfg.get("oids", []), state["readings"])}) - devices.sort(key=lambda d: _REACH_ORDER[d["reach"]]) - counts = {r: sum(1 for d in devices if d["reach"] == r) for r in _REACH_ORDER} - return await render_template("snmp/widget.html", - devices=devices[:DEVICES_SHOWN], - more=max(0, len(devices) - DEVICES_SHOWN), - counts=counts, total=len(devices)) + shown = [view(d, latest.get(d.name, {}), now) for d in devices] + shown.sort(key=lambda v: (_REACH_ORDER[v["reach"]], v["name"].lower())) + for v in shown: + v["key"] = [r for r in v["readings"] if r["text"] != "—"][:2] + counts = {r: sum(1 for d in shown if d["reach"] == r) for r in _REACH_ORDER} + return await render_template("snmp/widget.html", devices=shown[:DEVICES_SHOWN], + more=max(0, len(shown) - DEVICES_SHOWN), counts=counts, total=len(shown)) -@snmp_bp.get("/device/") +def devices_for_host(devices: list[SnmpDevice], host_id: str, address: str | None, + name: str | None) -> list[SnmpDevice]: + """The devices shown on a host's page: those bound to it, and those bound + to no host whose address is the host's address or name. A binding is + exclusive, so a device bound elsewhere never matches by address.""" + keys = {(address or "").strip().lower(), (name or "").strip().lower()} - {""} + return [d for d in devices + if (d.host_id == host_id if d.host_id else d.address.strip().lower() in keys)] + + +@snmp_bp.get("/host/") @require_role(UserRole.viewer) -async def device_detail(device_name: str): - devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", []) - device_cfg = next( - (d for d in devices_cfg if (d.get("name") or d.get("host")) == device_name), - None, - ) - if device_cfg is None: - from quart import abort - abort(404) - - hours = int(request.args.get("hours", 24)) +async def host_panel(host_id: str): + """The SNMP panel on a host's page; nothing when no device maps to it.""" + from steward.models.hosts import Host async with current_app.db_sessionmaker() as db: - hist = await _history(db, device_name, hours=hours) - - import json - history_json = json.dumps(hist) - oid_labels = [o.get("label") or o.get("oid") for o in device_cfg.get("oids", [])] - - return await render_template( - "snmp/device.html", - device=device_cfg, - device_name=device_name, - oid_labels=oid_labels, - history_json=history_json, - hours=hours, - ) + host = await db.get(Host, host_id) + if host is None: + return "" + matched = devices_for_host(await _all_devices(db), host.id, host.address, host.name) + if not matched: + return "" + latest = await _latest(db, [d.name for d in matched]) + now = datetime.now(timezone.utc) + return await render_template("snmp/host_panel.html", + devices=[{**view(d, latest.get(d.name, {}), now), "bound": bool(d.host_id)} + for d in matched]) diff --git a/plugins/snmp/scheduler.py b/plugins/snmp/scheduler.py index b9b1c79..70a8f7a 100644 --- a/plugins/snmp/scheduler.py +++ b/plugins/snmp/scheduler.py @@ -1,15 +1,23 @@ # plugins/snmp/scheduler.py from __future__ import annotations + +import asyncio import logging +from datetime import datetime, timezone from typing import TYPE_CHECKING +from sqlalchemy import select + +from steward.core.scheduler import ScheduledTask + if TYPE_CHECKING: from quart import Quart -from steward.core.scheduler import ScheduledTask - logger = logging.getLogger(__name__) +# Counters' previous samples, for their rates (service.RateKeeper). +RATES = None + def make_poll_task(app: "Quart") -> ScheduledTask: interval = app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60) @@ -25,46 +33,58 @@ def make_poll_task(app: "Quart") -> ScheduledTask: ) -async def _do_poll(app: "Quart") -> None: - from .poller import poll_device +async def _do_poll(app: "Quart", poll=None) -> None: + """Poll every device at once, then write each one's outcome to its row + and its answers to the metric tables. `poll` stands in for + poller.poll_device in tests.""" + global RATES from steward.core.alerts import MetricBatch - devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", []) + from .models import SnmpDevice + from .poller import poll_device + from .service import RateKeeper, credentials, readings_for + + poll = poll or poll_device + if RATES is None: + RATES = RateKeeper() + + async with app.db_sessionmaker() as session: + devices = (await session.execute(select(SnmpDevice))).scalars().all() if not devices: return + asked = {d.id: readings_for(d) for d in devices} + results = await asyncio.gather(*( + poll(d.address, d.port, credentials(d), [(r.oid, r.key) for r in asked[d.id]]) + for d in devices), return_exceptions=True) + + now = datetime.now(timezone.utc) async with app.db_sessionmaker() as session: async with session.begin(): batch = MetricBatch() - for device in devices: - if not isinstance(device, dict): + for device, result in zip(devices, results): + row = await session.get(SnmpDevice, device.id) + if row is None: # deleted while it was being polled continue - name = device.get("name") or device.get("host", "unknown") - host = device.get("host", "") - port = int(device.get("port", 161)) - community = device.get("community", "public") - version = str(device.get("version", "2c")) - oids = device.get("oids", []) - - if not host or not oids: + row.last_polled_at = now + if isinstance(result, BaseException): + logger.warning("SNMP poll of %s failed: %s", device.name, result) + row.last_error = f"SNMP poll failed: {result}" continue - - try: - readings = await poll_device(host, port, community, version, oids) - except Exception: - logger.exception("SNMP poll failed for device %s (%s)", name, host) - continue - - for label, value in readings.items(): - batch.add( - source_module="snmp", - resource_name=name, - metric_name=label, - value=value, - ) - - if readings: - logger.debug("SNMP polled %s (%s): %d OID(s)", name, host, len(readings)) - else: - logger.debug("SNMP polled %s (%s): no readings", name, host) + row.last_error = result.error + row.missing = sorted(set(result.missing)) + if result.readings: + row.last_ok_at = now + for reading in asked[device.id]: + value = result.readings.get(reading.key) + if value is None: + continue + if reading.counter: + value = RATES.rate(device.id, reading, value, now) + if value is None: + continue + else: + value = value * reading.scale + reading.offset + batch.add(source_module="snmp", resource_name=device.name, + metric_name=reading.key, value=value, recorded_at=now) await batch.record(session) diff --git a/plugins/snmp/service.py b/plugins/snmp/service.py new file mode 100644 index 0000000..bb91d7b --- /dev/null +++ b/plugins/snmp/service.py @@ -0,0 +1,195 @@ +# plugins/snmp/service.py +"""SNMP devices: the form a person edits, the row it becomes, what the poller +asks for, and what a page says about the newest poll.""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from steward.core.crypto import decrypt_secret, encrypt_secret + +from .models import SnmpDevice +from .poller import AUTH_PROTOCOLS, PRIV_PROTOCOLS, Credentials +from .readings import PRESETS, Reading, device_readings, parse_interfaces + +VERSIONS = ("1", "2c", "3") +PRESET_IDS = set(PRESETS) | {"interfaces"} +SECRET_FIELDS = ("community", "v3_auth_key", "v3_priv_key") + + +class FormError(ValueError): + """What is wrong with a submitted device form, said to the person.""" + + +def readings_for(device: SnmpDevice) -> list[Reading]: + """What the device is asked for. A row saved through the form always + parses; one that somehow doesn't is asked for nothing rather than raising + in the poll loop.""" + try: + return device_readings(device.presets or [], parse_interfaces(device.interfaces), + device.custom, device.version) + except ValueError: + return [] + + +def credentials(device: SnmpDevice) -> Credentials: + return Credentials( + version=device.version, + community=decrypt_secret(device.community, context=f"snmp device {device.name} community"), + user=device.v3_user, auth=device.v3_auth, + auth_key=decrypt_secret(device.v3_auth_key, context=f"snmp device {device.name} auth key"), + priv=device.v3_priv, + priv_key=decrypt_secret(device.v3_priv_key, context=f"snmp device {device.name} privacy key")) + + +def form_values(device: SnmpDevice | None) -> dict: + """The form's fields for an existing device. Secrets are never sent back + to the browser; a blank secret field on save keeps the stored one.""" + if device is None: + return {"port": 161, "version": "2c", "presets": ["system"], "v3_auth": "sha", "v3_priv": "aes128"} + return {"name": device.name, "address": device.address, "port": device.port, + "version": device.version, "v3_user": device.v3_user, "v3_auth": device.v3_auth, + "v3_priv": device.v3_priv, "presets": list(device.presets or []), + "interfaces": device.interfaces, "custom": device.custom, "host_id": device.host_id or "", + "has_community": bool(device.community), "has_auth_key": bool(device.v3_auth_key), + "has_priv_key": bool(device.v3_priv_key)} + + +def parse_form(form, existing: SnmpDevice | None = None) -> dict: + """The submitted form as the columns to store, secrets encrypted. + Raises FormError saying what to fix.""" + name = (form.get("name") or "").strip() + address = (form.get("address") or "").strip() + if not name: + raise FormError("Give the device a name.") + if len(name) > 100 or "/" in name: + raise FormError("A name is up to 100 characters, without a slash.") + if not address: + raise FormError("Give the device's address: an IP or a hostname.") + try: + port = int(form.get("port") or 161) + except ValueError: + raise FormError("The port is a number, usually 161.") from None + if not 1 <= port <= 65535: + raise FormError("The port is between 1 and 65535, usually 161.") + version = form.get("version") or "2c" + if version not in VERSIONS: + raise FormError("Choose SNMP version 1, 2c or 3.") + + presets = [p for p in form.getlist("presets") if p in PRESET_IDS] + interfaces = (form.get("interfaces") or "").strip() + custom = (form.get("custom") or "").strip() + try: + numbers = parse_interfaces(interfaces) + if "interfaces" in presets and not numbers: + raise FormError("Interface traffic needs the interface numbers to read, e.g. 1, 2 or 1-4.") + if not device_readings(presets, numbers, custom, version): + raise FormError("Choose at least one preset or add a custom reading.") + except ValueError as exc: + raise FormError(str(exc)) from None + + def secret(field: str) -> str: + typed = form.get(field) or "" + if typed: + return encrypt_secret(typed) + return getattr(existing, field) if existing is not None else "" + + values = {"name": name, "address": address, "port": port, "version": version, + "presets": presets, "interfaces": interfaces, "custom": custom, + "host_id": (form.get("host_id") or "").strip() or None, + "community": "", "v3_user": "", "v3_auth": "none", "v3_auth_key": "", + "v3_priv": "none", "v3_priv_key": ""} + if version == "3": + user = (form.get("v3_user") or "").strip() + auth = form.get("v3_auth") or "none" + priv = form.get("v3_priv") or "none" + if not user: + raise FormError("SNMPv3 needs a user name.") + if auth not in AUTH_PROTOCOLS or priv not in PRIV_PROTOCOLS: + raise FormError("Choose an authentication and a privacy protocol from the lists.") + if priv != "none" and auth == "none": + raise FormError("Privacy needs authentication too: choose an authentication protocol.") + values.update(v3_user=user, v3_auth=auth, v3_priv=priv, + v3_auth_key=secret("v3_auth_key") if auth != "none" else "", + v3_priv_key=secret("v3_priv_key") if priv != "none" else "") + for field, proto in (("v3_auth_key", auth), ("v3_priv_key", priv)): + if proto != "none" and not values[field]: + raise FormError("SNMPv3 with " + ("authentication" if field == "v3_auth_key" else "privacy") + + " needs its key (at least 8 characters).") + if proto != "none" and form.get(field) and len(form.get(field)) < 8: + raise FormError("An SNMPv3 key is at least 8 characters.") + else: + values["community"] = secret("community") or encrypt_secret("public") + return values + + +def form_credentials(form, existing: SnmpDevice | None = None) -> tuple[str, int, Credentials]: + """Address, port and credentials from an unsaved form, for Test. A blank + secret falls back to the stored one when testing an existing device.""" + def secret(field: str, default: str = "") -> str: + typed = form.get(field) or "" + if typed: + return typed + if existing is not None and getattr(existing, field): + return decrypt_secret(getattr(existing, field)) + return default + + version = form.get("version") if form.get("version") in VERSIONS else "2c" + try: + port = int(form.get("port") or 161) + except ValueError: + port = 161 + creds = Credentials(version=version, community=secret("community", "public"), + user=(form.get("v3_user") or "").strip(), auth=form.get("v3_auth") or "none", + auth_key=secret("v3_auth_key"), priv=form.get("v3_priv") or "none", + priv_key=secret("v3_priv_key")) + return (form.get("address") or "").strip(), port, creds + + +# A device not polled for this many intervals has stopped being polled +# (plugin turned off, scheduler stuck), whatever its last outcome said. +STALE_POLLS = 3 + + +@dataclass +class DeviceState: + reach: str # up | down | stale | pending + says: str # one line, in words + + +def device_state(device: SnmpDevice, poll_seconds: int, now: datetime) -> DeviceState: + """What the newest poll says about the device.""" + if device.last_polled_at is None: + return DeviceState("pending", "Not polled yet.") + if (now - device.last_polled_at).total_seconds() > STALE_POLLS * max(int(poll_seconds), 1): + return DeviceState("stale", "Not polled recently: the SNMP plugin may be turned off.") + if device.last_error: + return DeviceState("down", device.last_error) + return DeviceState("up", "Answering.") + + +class RateKeeper: + """A counter's previous sample per (device, key), to turn the next one + into a per-second rate. In memory: after a restart the first poll of each + counter stores nothing and the second stores its rate. A sample lower + than the one before (the device restarted, or a 32-bit counter wrapped) + is skipped rather than read as a huge negative or wrapped rate.""" + + def __init__(self) -> None: + self._last: dict[tuple[str, str], tuple[float, datetime]] = {} + + def rate(self, device_id: str, reading: Reading, value: float, at: datetime) -> float | None: + from .readings import COUNTER_UNITS + key = (device_id, reading.key) + prev = self._last.get(key) + self._last[key] = (value, at) + if prev is None: + return None + seconds = (at - prev[1]).total_seconds() + if seconds <= 0 or value < prev[0]: + return None + return (value - prev[0]) / seconds * COUNTER_UNITS[reading.unit][1] + + def forget(self, device_id: str) -> None: + for key in [k for k in self._last if k[0] == device_id]: + del self._last[key] diff --git a/plugins/snmp/templates/snmp/_test.html b/plugins/snmp/templates/snmp/_test.html new file mode 100644 index 0000000..3a02e28 --- /dev/null +++ b/plugins/snmp/templates/snmp/_test.html @@ -0,0 +1,22 @@ +{# What Test found (milestone 491): who the device says it is and its + interfaces with their numbers, or why it didn't answer. #} +{% if found.error %} + +{% else %} +
+

{{ address }} answered.

+
+
Name
{{ found.name or "—" }}
+
Description
{{ found.description or "—" }}
+
Up for
{{ (found.uptime_cs / 100)|duration if found.uptime_cs is not none else "—" }}
+
+ {% if found.interfaces %} +

Interfaces, by the number to put in Interface traffic:

+
+ {% for number, name in found.interfaces %}{{ number }} {{ name }}{% endfor %} +
+ {% else %} +

It lists no interfaces.

+ {% endif %} +
+{% endif %} diff --git a/plugins/snmp/templates/snmp/device.html b/plugins/snmp/templates/snmp/device.html index cc51aba..77dd7fd 100644 --- a/plugins/snmp/templates/snmp/device.html +++ b/plugins/snmp/templates/snmp/device.html @@ -1,82 +1,83 @@ -{# plugins/snmp/templates/snmp/device.html #} +{# plugins/snmp/templates/snmp/device.html — one SNMP device (milestone 491): + what its newest poll found, then a small chart per reading in that + reading's own unit, on a dated axis. #} {% extends "base.html" %} -{% from "_macros.html" import crumbs %} -{% block title %}SNMP — {{ device_name }} — Steward{% endblock %} -{% block breadcrumb %}{{ crumbs([("SNMP", "/plugins/snmp/"), (device_name, "")]) }}{% endblock %} +{% from "_macros.html" import crumbs, page_header, status_badge %} +{% block title %}SNMP — {{ device.name }} — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("SNMP", "/plugins/snmp/"), (device.name, "")]) }}{% endblock %} {% block content %} -
-

{{ device_name }}

- {{ device.host }} -
- {% for h in [1, 6, 24, 168] %} - - {% if h < 24 %}{{ h }}h{% elif h == 24 %}24h{% else %}7d{% endif %} - - {% endfor %} -
-
+{% call page_header(device.name, device.address ~ ((":" ~ device.port) if device.port != 161 else "") ~ " · SNMP v" ~ device.version) %} + {% for h in ranges %} + {% if h < 168 %}{{ h }}h{% else %}7d{% endif %} + {% endfor %} + {% if session.user_role == 'admin' %}Edit{% endif %} +{% endcall %} -{% if oid_labels %} -
- +

+ {% if device.reach == 'up' %}{{ status_badge('up', 'Answering') }} polled {{ device.polled_at|ago }} + {% elif device.reach == 'down' %}{{ status_badge('down', 'Not answering') }} {{ device.says }} + {% if device.ok_at %}Last answered {{ device.ok_at|ago }}.{% else %}It has never answered.{% endif %} + {% elif device.reach == 'stale' %}{{ status_badge('stale', 'Not polled') }} {{ device.says }} + {% else %}{{ status_badge('pending', 'Not polled yet') }} The first poll runs within a minute of saving.{% endif %} +

+{% if device.missing %}

Not answered by this device: {{ device.missing|join(", ") }}.

{% endif %} + +{% if charts %} +
+ {% for c in charts %} +
+
+

{{ c.name }}

+ {{ c.now }} +
+
+ +
+
+ {% endfor %}
+{% else %} +
This device has no readings chosen{% if session.user_role == 'admin' %}; Edit to choose some{% endif %}.
+{% endif %} -{% else %} -
- No OIDs configured for this device. -
-{% endif %} {% endblock %} diff --git a/plugins/snmp/templates/snmp/form.html b/plugins/snmp/templates/snmp/form.html new file mode 100644 index 0000000..20832d0 --- /dev/null +++ b/plugins/snmp/templates/snmp/form.html @@ -0,0 +1,166 @@ +{# plugins/snmp/templates/snmp/form.html — add or edit an SNMP device + (milestone 491): where it is, how to sign in, what to read, and Test to + ask the device before saving. Secrets are never sent back to the browser; + a blank secret field keeps the stored one. #} +{% extends "base.html" %} +{% from "_macros.html" import crumbs, page_header %} +{% block title %}{{ ("Edit " ~ device.name) if device else "Add SNMP device" }} — Steward{% endblock %} +{% block breadcrumb %} +{%- if device -%} +{{ crumbs([("SNMP", "/plugins/snmp/"), (device.name, "/plugins/snmp/devices/" ~ device.id), ("Edit", "")]) }} +{%- else -%} +{{ crumbs([("SNMP", "/plugins/snmp/"), ("Add device", "")]) }} +{%- endif -%} +{% endblock %} +{% block content %} +{{ page_header(("Edit " ~ device.name) if device else "Add SNMP device") }} + +
+{% if error %}{% endif %} +
{{ csrf_field() }} + {% if device %}{% endif %} + +
+

Device

+
+
+ + +
+
+ + +
+
+ + +
+
+ + + Which host's page shows this device's readings. +
+
+
+ +
+

Sign-in

+
+
+ + +
+
+ + + The device's read community. Blank {{ "keeps the saved one" if v.has_community else "means public" }}. +
+
+
+
+ + +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + Asking the device… + Asks the device for its name, uptime and interfaces, with what's typed here. Nothing is saved. +
+
+
+ +
+

Readings

+
+ {% for pid, label, what in presets %} + + {% if pid == "interfaces" %} +
+ + +
+ {% endif %} + {% endfor %} +
+
+ + + One a line: the OID, a name (letters, digits, underscores), and optionally a unit: + {% for u, words in units.items() %}{{ u }} ({{ words }}){{ ", " if not loop.last else "." }}{% endfor %} +
+
+ +
+ + Cancel +
+
+ +{% if device %} +
+
+

Delete this device

+

Steward stops polling it. Its past readings age out with the rest of the history.

+
+
{{ csrf_field() }} + +
+
+{% endif %} +
+ + +{% endblock %} diff --git a/plugins/snmp/templates/snmp/host_panel.html b/plugins/snmp/templates/snmp/host_panel.html index fb944f3..22677a8 100644 --- a/plugins/snmp/templates/snmp/host_panel.html +++ b/plugins/snmp/templates/snmp/host_panel.html @@ -1,52 +1,27 @@ -{# Per-host SNMP fragment — embedded on the Hosts hub via HTMX. Self-contained. - Shows the SNMP device(s) whose address maps to this host + latest readings. #} +{# plugins/snmp/templates/snmp/host_panel.html — the SNMP panel on a host's + page: the devices bound to this host, or at its address, with their + readings in their units, or why they aren't answering. #} +{% from "_macros.html" import status_badge, stat_tile %}
- - {% for device in devices %} -
+ {% for d in devices %} +
- {{ device.name }} - {{ device.host }} - {% if device.bound %} - bound - {% endif %} - {% if device.reach == 'up' %} - ● reachable · read {{ device.at|ago }} - {% elif device.reach == 'stale' %} - ● no answer since {{ device.at|ago }} - {% else %} - ○ no data yet - {% endif %} - History + {{ d.name }} + {{ d.address }} + {% if d.reach == 'up' %}{{ status_badge('up', 'Answering') }} + {% elif d.reach == 'down' %}{{ status_badge('down', 'Not answering', title=d.says) }} + {% elif d.reach == 'stale' %}{{ status_badge('stale', 'Not polled', title=d.says) }} + {% else %}{{ status_badge('pending', 'Not polled yet') }}{% endif %}
- - {% if device.readings %} -
- {% for oid_cfg in device.oids %} - {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %} - {% set val = device.readings.get(label) %} -
-
{{ label }}
- {% if val is not none %} -
- {% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}M - {% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}K - {% else %}{{ "%.4g"|format(val) }}{% endif %} -
- {% else %} -
—
- {% endif %} -
- {% endfor %} + {% if d.reach == 'down' %}

{{ d.says }}

{% endif %} + {% if d.reach != 'pending' and d.readings %} +
+ {% for r in d.readings %}{{ stat_tile(r.name, r.text) }}{% endfor %}
- {% else %} -

No readings yet — waiting for the next poll.

{% endif %}
{% endfor %} diff --git a/plugins/snmp/templates/snmp/index.html b/plugins/snmp/templates/snmp/index.html index 8b551f2..9f150b6 100644 --- a/plugins/snmp/templates/snmp/index.html +++ b/plugins/snmp/templates/snmp/index.html @@ -1,45 +1,55 @@ -{# plugins/snmp/templates/snmp/index.html #} +{# plugins/snmp/templates/snmp/index.html — every SNMP device, those not + answering first, each saying what its newest poll found: its readings in + their units, or why it didn't answer (milestone 491). #} {% extends "base.html" %} {% from "_macros.html" import page_header, status_badge, stat_tile, empty_state %} {% block title %}SNMP — Steward{% endblock %} {% block content %} -{{ page_header("SNMP devices", devices|length ~ " device" ~ ("s" if devices|length != 1 else "") ~ " configured") }} +{% set admin = session.user_role == 'admin' %} +{% call page_header("SNMP devices", (devices|length ~ " device" ~ ("s" if devices|length != 1 else "")) if devices else none) %} + {% if admin %}Add device{% endif %} +{% endcall %} {% if not devices %} -
{{ empty_state("No SNMP devices configured. Add them to the snmp plugin's plugin.yaml.") }}
-{% else %} +
{{ empty_state("No SNMP devices yet. Add a switch, a UPS, a printer or a server that answers SNMP, and Steward polls it every minute.", + "Add device" if admin else none, "/plugins/snmp/devices/new") }}
+{% endif %} -{% for device in devices %} -
+{% for d in devices %} +
- {#- reach: answered within the last few polls (up), only long ago (stale), - or never (pending); routes._device_state. #} - {% if device.reach == 'up' %}{{ status_badge('up', 'Reachable') }} - {% elif device.reach == 'stale' %}{{ status_badge('stale', 'No answer', title="The last reading is from " ~ device.at.strftime('%Y-%m-%d %H:%M UTC') ~ "; the values below are from then") }} - {% else %}{{ status_badge('pending', 'No data yet') }}{% endif %} -

{{ device.name }}

- {{ device.host }} - {% if device.at %}read {{ device.at|ago }}{% endif %} - History + {% if d.reach == 'up' %}{{ status_badge('up', 'Answering') }} + {% elif d.reach == 'down' %}{{ status_badge('down', 'Not answering') }} + {% elif d.reach == 'stale' %}{{ status_badge('stale', 'Not polled') }} + {% else %}{{ status_badge('pending', 'Not polled yet') }}{% endif %} +

{{ d.name }}

+ {{ d.address }}{% if d.port != 161 %}:{{ d.port }}{% endif %} + SNMP v{{ d.version }} + + History + {% if admin %}Edit{% endif %} +
- {% if device.readings %} + {% if d.reach == 'down' %} +

{{ d.says }}

+

+ {%- if d.ok_at %}Last answered {{ d.ok_at|ago }}; the readings below are from then.{% else %}It has never answered.{% endif %} + Tried {{ d.polled_at|ago }}.

+ {% elif d.reach == 'stale' %} +

{{ d.says }} Last polled {{ d.polled_at|ago }}.

+ {% elif d.reach == 'pending' %} +

Not polled yet: the first poll runs within a minute of saving.

+ {% endif %} + + {% if d.reach != 'pending' and d.readings %}
- {% for oid_cfg in device.oids %} - {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %} - {% set val = device.readings.get(label) %} - {{ stat_tile(label, - "—" if val is none else - ("%.2fM"|format(val / 1000000)) if val >= 1000000 else - ("%.1fK"|format(val / 1000)) if val >= 1000 else - "%.4g"|format(val)) }} - {% endfor %} + {% for r in d.readings %}{{ stat_tile(r.name, r.text, hint="from the second poll" if r.counter and r.text == "—" else none) }}{% endfor %}
- {% else %} -

No readings yet; waiting for the next poll.

+ {% endif %} + {% if d.missing %} +

Not answered by this device: {{ d.missing|join(", ") }}. It may not support them; Edit to remove them.

{% endif %}
{% endfor %} - -{% endif %} {% endblock %} diff --git a/plugins/snmp/templates/snmp/widget.html b/plugins/snmp/templates/snmp/widget.html index ca6b787..8955e19 100644 --- a/plugins/snmp/templates/snmp/widget.html +++ b/plugins/snmp/templates/snmp/widget.html @@ -1,15 +1,15 @@ -{# plugins/snmp/templates/snmp/widget.html — the dashboard SNMP widget (#5255): - a verdict, devices that stopped answering first with since when, then the - rest, each with its status in words and its key readings, named and with - units. The device name links to its page. #} +{# plugins/snmp/templates/snmp/widget.html — the dashboard SNMP widget (#5255, + milestone 491): a verdict, devices not answering first with why, then the + rest, each with two readings in their units. The name links to its page. #} {% from "_macros.html" import status_dot %} {% if not total %} -
No SNMP devices configured.
+
No SNMP devices yet.{% if session.user_role == 'admin' %} Add one{% endif %}
{% else %} -{% set stale = devices | selectattr("reach", "equalto", "stale") | list %} -
- {%- if counts.stale == 1 and stale %}{{ stale[0].name }}: no reply since {{ stale[0].at|ago }}. - {%- elif counts.stale %}{{ counts.stale }} of {{ total }} devices not answering. +{% set down = devices | selectattr("reach", "equalto", "down") | list %} +
+ {%- if counts.down == 1 and down %}{{ down[0].name }} isn't answering. + {%- elif counts.down %}{{ counts.down }} of {{ total }} devices not answering. + {%- elif counts.stale %}{{ counts.stale }} not polled recently. {%- elif counts.pending %}{{ counts.pending }} not polled yet, {{ counts.up }} answering. {%- else %}All {{ total }} device{{ 's' if total != 1 }} answering.{% endif -%}
@@ -20,15 +20,16 @@ {% for d in devices %} - {{- status_dot('down' if d.reach == 'stale' else ('pending' if d.reach == 'pending' else 'up')) -}} - {% if session.user_id %}{{ d.name }}{% else %}{{ d.name }}{% endif %} - {%- if d.reach == 'stale' %}no reply since {{ d.at|ago }}; readings are from then - {%- elif d.reach == 'pending' %}no data yet{% endif %} + {{- status_dot({'down': 'down', 'stale': 'stale', 'pending': 'pending'}.get(d.reach, 'up')) -}} + {% if session.user_id %}{{ d.name }}{% else %}{{ d.name }}{% endif %} + {%- if d.reach == 'down' %}{% if d.ok_at %}no reply since {{ d.ok_at|ago }}{% else %}never answered{% endif %} + {%- elif d.reach == 'stale' %}not polled since {{ d.polled_at|ago }} + {%- elif d.reach == 'pending' %}not polled yet{% endif %} - - {%- for n, v in d.key %}{{ n }} {{ v }}{% if not loop.last %} · {% endif %}{% else %}—{% endfor -%} + + {%- for r in d.key %}{{ r.name }} {{ r.text }}{% if not loop.last %} · {% endif %}{% else %}—{% endfor -%} - {% if d.at %}{{ d.at|ago }}{% else %}—{% endif %} + {% if d.polled_at %}{{ d.polled_at|ago }}{% else %}—{% endif %} {% endfor %} diff --git a/steward/audit/service.py b/steward/audit/service.py index adde7b5..3cf6175 100644 --- a/steward/audit/service.py +++ b/steward/audit/service.py @@ -6,6 +6,8 @@ An entity deleted since keeps whatever name the entry's own detail recorded. """ from __future__ import annotations +import logging + from sqlalchemy import select from steward.ansible.inventory_gen import run_title, scope_labels @@ -18,6 +20,8 @@ from steward.models.api_tokens import ApiToken from steward.models.hosts import Host from steward.models.monitors import Monitor +logger = logging.getLogger(__name__) + # entity_type -> (model, its page for an id). A type named by a value rather # than a row (plugin, ansible_playbook) is its own label. _NAMED = { @@ -33,6 +37,23 @@ _NAMED = { _SELF_NAMED = {"plugin": "/settings/plugins/", "ansible_playbook": None} +def _named_types() -> dict: + """Core's named entity types plus those a loaded plugin names through its + get_audit_entities() hook ({type: (model, page for an id)}), so core + reads a plugin's rows without importing it.""" + from steward.core.plugin_manager import get_loaded_plugin_modules + named = dict(_NAMED) + for name, module in get_loaded_plugin_modules().items(): + hook = getattr(module, "get_audit_entities", None) + if hook is None: + continue + try: + named.update(hook() or {}) + except Exception: + logger.exception("Plugin %r: get_audit_entities() raised; its entries show by id", name) + return named + + def _recorded_name(detail: dict) -> str | None: """A name the entry recorded when it was written, for a deleted entity.""" for key in ("name", "new_name", "username", "playbook"): @@ -50,10 +71,11 @@ async def entity_labels(db, items: list[dict]) -> dict[tuple[str, str], dict]: if e.entity_type and e.entity_id: ids.setdefault(e.entity_type, set()).add(str(e.entity_id)) + named = _named_types() labels: dict[tuple[str, str], dict] = {} for etype, wanted in ids.items(): - if etype in _NAMED: - model, page = _NAMED[etype] + if etype in named: + model, page = named[etype] key_col = model.id keys = [int(i) for i in wanted if i.isdigit()] if etype == "api_token" else list(wanted) if not keys: @@ -142,6 +164,9 @@ ACTIONS = { "api_token.create": "created AI access token", "api_token.revoke": "revoked AI access token", "docker.prune.started": "started a Docker cleanup on host", + "snmp.device_created": "added SNMP device", + "snmp.device_updated": "changed SNMP device", + "snmp.device_deleted": "deleted SNMP device", } # The filter's choices: an area, matched as a prefix of the action code. @@ -149,7 +174,7 @@ AREAS = (("auth.", "Sign-ins"), ("account.", "Accounts"), ("host.", "Hosts"), (" ("alert_rule.", "Alert rules"), ("maintenance_window.", "Maintenance"), ("ansible.run", "Playbook runs"), ("ansible.", "Ansible (all)"), ("settings.", "Settings"), ("plugin.", "Plugins"), ("api_token.", "AI access"), - ("docker.", "Docker")) + ("docker.", "Docker"), ("snmp.", "SNMP")) def action_text(action: str) -> str: diff --git a/steward/static/steward.css b/steward/static/steward.css index 83dc999..7f6143f 100644 --- a/steward/static/steward.css +++ b/steward/static/steward.css @@ -287,6 +287,7 @@ select { cursor: pointer; } textarea { resize: vertical; } /* Form fields: .field wraps a .field-label, the .input and an optional .field-hint. */ .field { display: flex; flex-direction: column; gap: 0.2rem; margin-bottom: 0.9rem; } +.field[hidden] { display: none; } .field-label { display: block; margin-bottom: 0; font-size: 0.75rem; color: var(--text-muted); } .field-hint { font-size: 0.75rem; color: var(--text-dim); } .field-error { font-size: 0.85rem; color: var(--red); margin-bottom: 0.75rem; } diff --git a/tests/integration/test_snmp_poll.py b/tests/integration/test_snmp_poll.py new file mode 100644 index 0000000..58c05d8 --- /dev/null +++ b/tests/integration/test_snmp_poll.py @@ -0,0 +1,76 @@ +"""Integration: an SNMP poll writes each device's outcome to its row and its +answers to the metric tables, a counter as a rate (milestone 491). +Requires STEWARD_DATABASE_URL.""" +from __future__ import annotations + +import asyncio +import os +import uuid + +import pytest + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +def test_a_poll_records_why_a_device_is_silent_and_what_another_answered(app): + from sqlalchemy import select + + from plugins.snmp import scheduler + from plugins.snmp.models import SnmpDevice + from plugins.snmp.poller import PollResult + from steward.models.metrics import PluginMetricLatest + + tag = uuid.uuid4().hex[:8] + up, down = f"sw-{tag}", f"ups-{tag}" + counts = iter([1_000.0, 61_000.0]) + + async def fake_poll(address, port, creds, oids): + if address == "10.9.9.9": + return PollResult(error="No reply from 10.9.9.9:161 within 5 s.") + keys = dict((k, o) for o, k in oids) + assert {"uptime", "if1_in", "if1_status"} <= set(keys) and creds.community == "public" + return PollResult(readings={"uptime": 8_640_000.0, "if1_status": 1.0, "if1_in": next(counts)}, + missing=["if1_out"]) + + async def go(): + from steward.core.crypto import encrypt_secret + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(SnmpDevice(name=up, address="10.9.9.1", presets=["system", "interfaces"], + interfaces="1", community=encrypt_secret("public"))) + s.add(SnmpDevice(name=down, address="10.9.9.9", presets=["ups"], + community=encrypt_secret("public"))) + scheduler.RATES = None + await scheduler._do_poll(app, poll=fake_poll) + first = await _latest(app, up) + await asyncio.sleep(1.1) + await scheduler._do_poll(app, poll=fake_poll) + async with app.db_sessionmaker() as s: + rows = {d.name: d for d in (await s.execute( + select(SnmpDevice).where(SnmpDevice.name.in_([up, down])))).scalars()} + async with s.begin(): + for d in rows.values(): + await s.delete(d) + return first, await _latest(app, up), rows + + async def _latest(app, name): + async with app.db_sessionmaker() as s: + return {r.metric_name: r.value for r in (await s.execute(select(PluginMetricLatest).where( + PluginMetricLatest.source_module == "snmp", PluginMetricLatest.resource_name == name))).scalars()} + + first, second, rows = asyncio.run(go()) + assert rows[up].last_error is None and rows[up].last_ok_at is not None and rows[up].missing == ["if1_out"] + assert rows[down].last_error == "No reply from 10.9.9.9:161 within 5 s." and rows[down].last_ok_at is None + assert rows[down].last_polled_at is not None + assert first == {"uptime": 8_640_000.0, "if1_status": 1.0} # a counter needs two samples + # 60,000 bytes over the 1.1 s or more between the polls, stored as bits + # per second: at most 436 kbit/s, and above 150 unless a poll took seconds. + assert 150_000 < second["if1_in"] <= 436_364 diff --git a/tests/plugins/snmp/test_host_match.py b/tests/plugins/snmp/test_host_match.py deleted file mode 100644 index 7c02afb..0000000 --- a/tests/plugins/snmp/test_host_match.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Unit tests for mapping config-defined SNMP devices onto a Steward host.""" -from plugins.snmp.routes import _devices_for_host - -_CFG = [ - {"name": "core-switch", "host": "192.168.1.1"}, - {"name": "ups", "host": "192.168.1.10"}, - {"name": "by-name", "host": "edge-router"}, - "not-a-dict", # tolerated, skipped - {"name": "no-host"}, # no host → never matches -] - - -def test_matches_by_address(): - out = _devices_for_host(_CFG, "192.168.1.1", "somehost") - assert [d["name"] for d in out] == ["core-switch"] - - -def test_matches_by_name_case_insensitive(): - out = _devices_for_host(_CFG, "10.0.0.9", "Edge-Router") - assert [d["name"] for d in out] == ["by-name"] - - -def test_no_match_returns_empty(): - assert _devices_for_host(_CFG, "10.0.0.1", "nope") == [] - - -def test_blank_address_and_name_match_nothing(): - # A host with no address/name must not match a device with a blank host. - assert _devices_for_host(_CFG, "", None) == [] - - -def test_implicit_match_unaffected_when_host_id_passed(): - # Passing the new host_id arg must not break the legacy implicit match. - out = _devices_for_host(_CFG, "192.168.1.1", "somehost", host_id="h-1") - assert [d["name"] for d in out] == ["core-switch"] - - -# ── Explicit binding ───────────────────────────────────────────────────────── - -def test_explicit_host_id_matches_exact_uuid(): - cfg = [{"name": "sw", "host": "10.0.0.5", "host_id": "uuid-abc"}] - out = _devices_for_host(cfg, "10.0.0.9", "other", host_id="uuid-abc") - assert [d["name"] for d in out] == ["sw"] - - -def test_explicit_steward_host_matches_by_name_or_address(): - cfg = [{"name": "pdu", "host": "10.0.0.5", "steward_host": "Switch01"}] - # Bind hits the Host's name despite a different poll target / address. - by_name = _devices_for_host(cfg, "10.0.0.9", "switch01", host_id="h9") - by_addr = _devices_for_host(cfg, "switch01", "whatever", host_id="h9") - assert [d["name"] for d in by_name] == ["pdu"] - assert [d["name"] for d in by_addr] == ["pdu"] - - -def test_explicit_binding_is_exclusive(): - # Device polls 192.168.1.1 but is explicitly bound to host A by name. - cfg = [{"name": "sw", "host": "192.168.1.1", "steward_host": "hostA"}] - # Host B *would* match implicitly by the poll target — but must NOT, - # because the explicit binding takes over and points only at host A. - hostB = _devices_for_host(cfg, "192.168.1.1", "hostB", host_id="b") - hostA = _devices_for_host(cfg, "10.0.0.2", "hostA", host_id="a") - assert hostB == [] - assert [d["name"] for d in hostA] == ["sw"] - - -def test_explicit_host_id_wrong_uuid_does_not_match(): - cfg = [{"name": "sw", "host": "10.0.0.5", "host_id": "uuid-abc"}] - out = _devices_for_host(cfg, "10.0.0.5", "sw", host_id="uuid-zzz") - assert out == [] diff --git a/tests/plugins/snmp/test_poller.py b/tests/plugins/snmp/test_poller.py index ab64701..626fdf5 100644 --- a/tests/plugins/snmp/test_poller.py +++ b/tests/plugins/snmp/test_poller.py @@ -1,44 +1,125 @@ -"""Unit tests for the SNMP poller (no pysnmp / no network). +"""The SNMP poller (milestone 491), against a stand-in for pysnmp. -The unit lane doesn't install the `snmp` extra, so these exercise the parts that -don't need pysnmp: the version→mpModel mapping, that the poller is async, and the -graceful "pysnmp missing → empty result" path. Live polling against a real device -is verified out-of-band (CI has no SNMP target). Uses asyncio.run() directly so it -doesn't depend on the pytest-asyncio mode. +The unit lane doesn't install the `snmp` extra, so a fake API answers in +pysnmp's shape: one GET per device, answers in request order, an agent's "no +such object" as a value of that type, v1's whole-request error with an index, +and a timeout as an error indication. Live polling is read from the instance. """ import asyncio import inspect +from types import SimpleNamespace as NS from plugins.snmp import poller +from plugins.snmp.poller import Credentials -def test_poll_device_is_coroutine(): - # The scheduler awaits it directly (no executor) — it must be async. +class NoSuchObject: + pass + + +class _Status: + def __init__(self, text): + self.text = text + + def __bool__(self): + return True + + def prettyPrint(self): + return self.text + + +def _fake_api(answer, calls): + """answer(oids) -> (error_indication, error_status, error_index, values).""" + async def get(engine, auth, transport, ctx, *objs, **opts): + oids = [o for o in objs] + calls.append((auth, oids, opts)) + ei, es, eidx, values = answer(oids) + return ei, es, eidx, [(oid, v) for oid, v in zip(oids, values)] + + class Transport: + def __init__(self, addr, **kw): + self.addr = addr + + hl = NS(SnmpEngine=lambda: NS(), ContextData=lambda: "ctx", ObjectType=lambda x: x, + ObjectIdentity=lambda oid: oid, UdpTransportTarget=Transport, + CommunityData=lambda c, mpModel: ("community", c, mpModel), + UsmUserData=lambda user, **kw: ("usm", user, kw)) + auth = NS(**{name: name for name in list(poller.AUTH_PROTOCOLS.values()) + list(poller.PRIV_PROTOCOLS.values())}) + return NS(hl=hl, auth=auth, get=get, walk=None, async_transport=False) + + +def _poll(monkeypatch, answer, oids, creds=None): + calls = [] + monkeypatch.setattr(poller, "_pysnmp_available", lambda: True) + monkeypatch.setattr(poller, "_api", lambda: _fake_api(answer, calls)) + result = asyncio.run(poller.poll_device("10.0.0.2", 161, creds or Credentials(community="public"), oids)) + return result, calls + + +def test_poll_device_is_a_coroutine(): assert inspect.iscoroutinefunction(poller.poll_device) -def test_mp_model_maps_version_to_int(): - assert poller._mp_model("1") == 0 # SNMP v1 - assert poller._mp_model("2c") == 1 # SNMP v2c - assert poller._mp_model("2") == 1 # anything non-"1" → v2c model +def test_one_get_carries_every_oid_and_answers_come_back_by_key(monkeypatch): + result, calls = _poll(monkeypatch, lambda oids: (None, None, None, [4200.0, NoSuchObject(), "eth0"]), + [("1.1", "uptime"), ("1.2", "fan"), ("1.3", "label")]) + assert len(calls) == 1 and calls[0][1] == ["1.1", "1.2", "1.3"] and calls[0][2] == {"lookupMib": False} + assert calls[0][0] == ("community", "public", 1) + assert result.readings == {"uptime": 4200.0} and result.missing == ["fan", "label"] and result.error is None -def test_poll_device_without_pysnmp_returns_empty(monkeypatch): - # When the optional dep is absent, polling is disabled gracefully (no raise). +def test_v1_drops_the_oid_it_does_not_know_and_asks_again(monkeypatch): + def answer(oids): + if "1.2" in oids: + return None, _Status("noSuchName"), oids.index("1.2") + 1, [] + return None, None, None, [float(len(o)) for o in oids] + result, calls = _poll(monkeypatch, answer, [("1.1", "a"), ("1.2", "b"), ("1.33", "c")], + Credentials(version="1", community="public")) + assert len(calls) == 2 and calls[0][0] == ("community", "public", 0) + assert result.readings == {"a": 3.0, "c": 4.0} and result.missing == ["b"] + + +def test_no_reply_says_what_to_check(monkeypatch): + result, _ = _poll(monkeypatch, lambda oids: ("No SNMP response received before timeout", None, None, []), + [("1.1", "uptime")]) + assert result.readings == {} and result.error.startswith("No reply from 10.0.0.2:161 within 5 s") + assert "the community" in result.error + + +def test_an_answer_without_any_reading_is_an_error(monkeypatch): + result, _ = _poll(monkeypatch, lambda oids: (None, None, None, [NoSuchObject()]), [("1.1", "x")]) + assert result.error == "The device answered but has none of these readings." and result.missing == ["x"] + + +def test_v3_signs_in_as_its_user_with_the_chosen_protocols(monkeypatch): + creds = Credentials(version="3", user="mon", auth="sha256", auth_key="authpass1", + priv="aes128", priv_key="privpass1") + _, calls = _poll(monkeypatch, lambda oids: (None, None, None, [1.0]), [("1.1", "x")], creds) + kind, user, kw = calls[0][0] + assert (kind, user) == ("usm", "mon") + assert kw == {"authKey": "authpass1", "privKey": "privpass1", + "authProtocol": "usmHMAC192SHA256AuthProtocol", "privProtocol": "usmAesCfb128Protocol"} + + +def test_without_pysnmp_the_poll_says_so(monkeypatch): monkeypatch.setattr(poller, "_pysnmp_available", lambda: False) - out = asyncio.run( - poller.poll_device("192.0.2.1", 161, "public", "2c", [{"oid": "1.3.6.1.2.1.1.3.0"}]) - ) - assert out == {} + result = asyncio.run(poller.poll_device("10.0.0.2", 161, Credentials(), [("1.1", "x")])) + assert result.readings == {} and "pysnmp" in result.error + + +def test_errors_read_as_what_to_check(): + v3 = Credentials(version="3", user="mon") + assert "doesn't know the v3 user “mon”" in poller.explain("Unknown USM user", "h", 161, v3) + assert "authentication" in poller.explain("Wrong SNMP PDU digest", "h", 161, v3) + assert "the v3 user “mon”" in poller.explain("No SNMP response received before timeout", "h", 161, v3) + assert poller.explain("Something odd", "h", 161, v3) == "SNMP error from h:161: Something odd" # --- _close_engine: the fd-leak guard --------------------------------------- # A fresh SnmpEngine per poll leaks its UDP socket unless closed; _close_engine -# must release it across both pysnmp API shapes. These fakes stand in for the -# engine since the unit lane has no pysnmp. +# must release it across both pysnmp API shapes. def test_close_engine_uses_engine_level_close_7x(): - # canonical pysnmp 7.x: close method lives directly on the engine. calls = [] class Engine: @@ -50,7 +131,6 @@ def test_close_engine_uses_engine_level_close_7x(): def test_close_engine_falls_back_to_dispatcher_6x(): - # pysnmp-lextudio 6.2.x: no engine-level close; go through the dispatcher. calls = [] class Dispatcher: @@ -65,12 +145,9 @@ def test_close_engine_falls_back_to_dispatcher_6x(): def test_close_engine_never_raises(): - # A failing close must not break the poll loop. class Engine: def close_dispatcher(self): raise OSError("boom") - poller._close_engine(Engine()) # no exception - - # No close path at all (defensive) is also fine. + poller._close_engine(Engine()) poller._close_engine(object()) diff --git a/tests/plugins/snmp/test_readings.py b/tests/plugins/snmp/test_readings.py new file mode 100644 index 0000000..5955fe5 --- /dev/null +++ b/tests/plugins/snmp/test_readings.py @@ -0,0 +1,67 @@ +"""What an SNMP device is asked for and how each answer reads (milestone 491).""" +import pytest + +from plugins.snmp.readings import (device_readings, format_value, interface_readings, parse_custom, + parse_interfaces, stored_unit) + + +def test_interface_numbers_read_as_a_list(): + assert parse_interfaces("1, 2 5-7,2") == [1, 2, 5, 6, 7] + assert parse_interfaces("") == [] + for bad in ("eth0", "7-3", "1-100"): + with pytest.raises(ValueError): + parse_interfaces(bad) + + +def test_interfaces_read_64_bit_counters_except_on_v1(): + v2 = {r.key: r for r in interface_readings([3], "2c")} + v1 = {r.key: r for r in interface_readings([3], "1")} + assert set(v2) == {"if3_status", "if3_in", "if3_out"} + assert v2["if3_in"].oid == "1.3.6.1.2.1.31.1.1.1.6.3" and v2["if3_in"].counter + assert v1["if3_in"].oid == "1.3.6.1.2.1.2.2.1.10.3" + assert stored_unit(v2["if3_out"]) == "bits_per_second" and not v2["if3_status"].counter + + +def test_a_custom_line_is_an_oid_a_name_and_a_unit(): + got = parse_custom("# a comment\n.1.3.6.1.4.1.9.1.0 fan_speed\n1.3.6.1.4.1.9.2.0 inlet_temp celsius\n") + assert [(r.oid, r.key, r.name, r.unit) for r in got] == [ + ("1.3.6.1.4.1.9.1.0", "fan_speed", "Fan speed", "number"), + ("1.3.6.1.4.1.9.2.0", "inlet_temp", "Inlet temp", "celsius")] + + +@pytest.mark.parametrize("line, says", [ + ("1.3.6.1", "OID name"), + ("sysUpTime uptime", "is not an OID"), + ("1.3.6.1.2 Bad-Name", "must start with a letter"), + ("1.3.6.1.2 temp furlongs", "is not a unit"), +]) +def test_a_custom_line_that_does_not_read_says_which_and_why(line, says): + with pytest.raises(ValueError, match="Line 1") as exc: + parse_custom(line) + assert says in str(exc.value) + + +def test_presets_come_in_order_and_names_may_not_repeat(): + keys = [r.key for r in device_readings(["ups", "system", "interfaces"], [1], "", "2c")] + assert keys[:4] == ["uptime", "if1_status", "if1_in", "if1_out"] and "battery_charge" in keys + with pytest.raises(ValueError, match="uptime"): + device_readings(["system"], [], "1.3.6.1.2.1.1.3.0 uptime", "2c") + + +@pytest.mark.parametrize("unit, value, text", [ + ("centiseconds", 105_120_000, "12d 4h"), + ("percent", 99.6, "100%"), + ("volts", 121.0, "121 V"), + ("decivolts", 1203, "120.3 V"), + ("minutes", 45, "45m"), + ("kilobytes", 2048, "2.0 MiB"), + ("bits_per_second", 12_345_678, "12.3 Mbit/s"), + ("bits_per_second", 950, "950 bit/s"), + ("if_status", 2, "down"), + ("ups_battery", 3, "low"), + ("if_status", 99, "code 99"), + ("number", 41.25, "41.25"), + ("number", None, "—"), +]) +def test_a_value_reads_in_its_unit(unit, value, text): + assert format_value(unit, value) == text diff --git a/tests/plugins/snmp/test_service.py b/tests/plugins/snmp/test_service.py new file mode 100644 index 0000000..e83a46e --- /dev/null +++ b/tests/plugins/snmp/test_service.py @@ -0,0 +1,118 @@ +"""The SNMP device form, the row it becomes, and what a page says about the +newest poll (milestone 491).""" +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace as NS + +import pytest +from werkzeug.datastructures import MultiDict + +from plugins.snmp.readings import Reading +from plugins.snmp.service import (FormError, RateKeeper, credentials, device_state, form_credentials, + form_values, parse_form, readings_for) +from steward.core.crypto import decrypt_secret, init_crypto, is_encrypted + + +def setup_function(): + init_crypto("unit-test-secret-key") + + +def _form(**fields): + presets = fields.pop("presets", ["system"]) + md = MultiDict({"name": "core-switch", "address": "192.168.1.2", "port": "161", "version": "2c", **fields}) + for p in presets: + md.add("presets", p) + return md + + +def test_a_device_is_stored_with_its_community_encrypted(): + values = parse_form(_form(community="s3cret", interfaces="1-2", presets=["system", "interfaces"])) + assert values["name"] == "core-switch" and values["port"] == 161 and values["version"] == "2c" + assert values["presets"] == ["system", "interfaces"] and values["interfaces"] == "1-2" + assert is_encrypted(values["community"]) and decrypt_secret(values["community"]) == "s3cret" + assert values["v3_user"] == "" and values["host_id"] is None + + +def test_a_blank_community_is_public_on_a_new_device_and_kept_on_an_old_one(): + assert decrypt_secret(parse_form(_form())["community"]) == "public" + existing = NS(community="enc:kept", v3_auth_key="", v3_priv_key="") + assert parse_form(_form(), existing)["community"] == "enc:kept" + + +@pytest.mark.parametrize("fields, says", [ + ({"name": ""}, "name"), + ({"address": ""}, "address"), + ({"port": "99999"}, "port"), + ({"version": "4"}, "version"), + ({"presets": []}, "at least one"), + ({"presets": ["interfaces"]}, "interface numbers"), + ({"custom": "1.3.6 Bad-Name"}, "Line 1"), + ({"version": "3"}, "user name"), + ({"version": "3", "v3_user": "mon", "v3_auth": "none", "v3_priv": "aes128"}, "Privacy needs authentication"), + ({"version": "3", "v3_user": "mon", "v3_auth": "sha", "v3_priv": "none"}, "needs its key"), + ({"version": "3", "v3_user": "mon", "v3_auth": "sha", "v3_auth_key": "short", "v3_priv": "none"}, + "at least 8"), +]) +def test_a_form_that_cannot_be_saved_says_what_to_fix(fields, says): + with pytest.raises(FormError, match=says): + parse_form(_form(**fields)) + + +def test_a_v3_device_keeps_its_user_and_encrypted_keys_and_no_community(): + values = parse_form(_form(version="3", v3_user="mon", v3_auth="sha256", v3_auth_key="authpass1", + v3_priv="aes128", v3_priv_key="privpass1")) + assert values["community"] == "" and values["v3_user"] == "mon" + assert (values["v3_auth"], values["v3_priv"]) == ("sha256", "aes128") + assert decrypt_secret(values["v3_auth_key"]) == "authpass1" and decrypt_secret(values["v3_priv_key"]) == "privpass1" + row = NS(name="x", **{k: values[k] for k in ("version", "community", "v3_user", "v3_auth", + "v3_auth_key", "v3_priv", "v3_priv_key")}) + creds = credentials(row) + assert (creds.user, creds.auth_key, creds.priv_key) == ("mon", "authpass1", "privpass1") + + +def test_the_edit_form_never_carries_a_secret_back(): + device = NS(name="ups", address="10.0.0.9", port=161, version="2c", v3_user="", v3_auth="none", + v3_priv="none", presets=["ups"], interfaces="", custom="", host_id=None, + community="enc:x", v3_auth_key="", v3_priv_key="") + v = form_values(device) + assert v["has_community"] and not v["has_auth_key"] + assert "enc:x" not in str(v) + + +def test_test_uses_what_is_typed_and_falls_back_to_the_stored_secret(): + from steward.core.crypto import encrypt_secret + existing = NS(community=encrypt_secret("stored"), v3_auth_key="", v3_priv_key="") + address, port, creds = form_credentials(_form(port="1161"), existing) + assert (address, port, creds.community) == ("192.168.1.2", 1161, "stored") + assert form_credentials(_form(community="typed"), existing)[2].community == "typed" + assert form_credentials(_form(), None)[2].community == "public" + + +def test_a_row_that_no_longer_parses_is_asked_for_nothing(): + row = NS(presets=["system"], interfaces="eth0", custom="", version="2c") + assert readings_for(row) == [] + + +def _row(**kw): + return NS(**{"last_polled_at": None, "last_error": None, **kw}) + + +def test_the_newest_poll_decides_what_the_page_says(): + now = datetime(2026, 10, 8, 15, 0, tzinfo=timezone.utc) + recent = now - timedelta(seconds=30) + assert device_state(_row(), 60, now).reach == "pending" + assert device_state(_row(last_polled_at=recent), 60, now).reach == "up" + down = device_state(_row(last_polled_at=recent, last_error="No reply from x"), 60, now) + assert (down.reach, down.says) == ("down", "No reply from x") + assert device_state(_row(last_polled_at=now - timedelta(minutes=5)), 60, now).reach == "stale" + + +def test_a_counter_becomes_a_rate_and_a_reset_is_skipped(): + keeper = RateKeeper() + rx = Reading("1.3", "if1_in", "Interface 1 in", "octets") + t0 = datetime(2026, 10, 8, 15, 0, tzinfo=timezone.utc) + assert keeper.rate("d1", rx, 1_000, t0) is None # nothing to compare yet + assert keeper.rate("d1", rx, 7_000, t0 + timedelta(seconds=60)) == 800.0 # 100 B/s as bit/s + assert keeper.rate("d1", rx, 50, t0 + timedelta(seconds=120)) is None # rebooted or wrapped + assert keeper.rate("d1", rx, 650, t0 + timedelta(seconds=180)) == 80.0 + keeper.forget("d1") + assert keeper.rate("d1", rx, 9_999, t0 + timedelta(seconds=240)) is None diff --git a/tests/plugins/snmp/test_snmp_page.py b/tests/plugins/snmp/test_snmp_page.py index 88ffbe8..e204883 100644 --- a/tests/plugins/snmp/test_snmp_page.py +++ b/tests/plugins/snmp/test_snmp_page.py @@ -1,39 +1,96 @@ -"""The SNMP list says each device's state with the shared badge and shows its -readings as stat tiles (#5126).""" +"""The SNMP pages (milestone 491): devices not answering first and saying why, +readings in their units, the editor, and Test's answer.""" from datetime import datetime, timedelta, timezone +from types import SimpleNamespace as NS -from plugins.snmp.routes import _device_state +from plugins.snmp.routes import devices_for_host from tests._templates import render_plugin_template from tests.test_style_tokens import hex_colours -async def test_devices_wear_a_status_badge_and_readings_are_tiles(app): +def _view(name, reach, says="Answering.", readings=(), ok_minutes=None, missing=()): + now = datetime.now(timezone.utc) + return {"id": f"id-{name}", "name": name, "address": "10.0.0.9", "port": 161, "version": "2c", + "reach": reach, "says": says, "polled_at": now - timedelta(seconds=20), + "ok_at": now - timedelta(minutes=ok_minutes) if ok_minutes is not None else None, + "readings": list(readings), "missing": list(missing)} + + +async def test_a_device_not_answering_says_why_and_since_when(app): + from quart import session devices = [ - {"name": "ups", "host": "10.0.0.9", "oids": [{"oid": "1.3.6", "label": "battery_charge_pct"}, - {"oid": "1.3.7", "label": "in_octets"}], - "readings": {"battery_charge_pct": 97.0, "in_octets": 2_500_000.0}, "reach": "up", - "at": datetime.now(timezone.utc) - timedelta(seconds=30)}, - {"name": "switch", "host": "10.0.0.11", "oids": [], "readings": {"x": 1.0}, "reach": "stale", - "at": datetime.now(timezone.utc) - timedelta(hours=3)}, - {"name": "printer", "host": "10.0.0.10", "oids": [], "readings": {}, "reach": "pending", "at": None}, + _view("ups", "down", "No reply from 10.0.0.9:161 within 5 s. Check the address.", ok_minutes=180, + readings=[{"key": "battery_charge", "name": "Charge", "counter": False, "text": "97%"}]), + _view("printer", "down", "No reply from 10.0.0.10:161 within 5 s."), + _view("switch", "up", readings=[{"key": "if1_in", "name": "Interface 1 in", "counter": True, "text": "—"}, + {"key": "uptime", "name": "Uptime", "counter": False, "text": "12d 4h"}], + missing=["Load"]), + _view("new", "pending"), ] async with app.test_request_context("/plugins/snmp/"): - page = await render_plugin_template(app, "snmp", "snmp/index.html", devices=devices, poll_interval=60) - - assert 'Reachable' in page - assert 'No data yet' in page - # Old readings are not "reachable": they say how old they are (#5210). - assert 'badge badge-warn" title="The last reading is from' in page and ">No answer" in page - assert ">3h ago" in page - assert '97' in page and '2.50M' in page - assert 'href="/plugins/snmp/device/ups"' in page + session["user_role"] = "admin" + page = await render_plugin_template(app, "snmp", "snmp/index.html", devices=devices) + page = " ".join(page.split()) + assert page.count('Not answering') == 2 + assert "No reply from 10.0.0.9:161 within 5 s. Check the address." in page + assert "Last answered" in page and ">3h ago" in page and "It has never answered." in page + assert "the readings below are from then" in page + assert '97%' in page and '12d 4h' in page + assert "from the second poll" in page # a counter's rate needs two polls + assert "Not answered by this device: Load" in page + assert "Not polled yet: the first poll runs within a minute of saving." in page + assert 'href="/plugins/snmp/devices/new"' in page and 'href="/plugins/snmp/devices/id-ups/edit"' in page main = page[page.index("49 xe-0/1/0' in ok + assert 'role="alert"' in bad and "No reply from 10.0.0.2:161" in bad + + +def test_a_device_shows_on_its_bound_host_or_at_its_address(): + devices = [NS(name="sw", address="192.168.1.2", host_id=None), + NS(name="ups", address="10.0.0.9", host_id="h-nas"), + NS(name="pdu", address="nas.lan", host_id=None)] + assert [d.name for d in devices_for_host(devices, "h-nas", "192.168.1.50", "NAS.lan")] == ["ups", "pdu"] + assert [d.name for d in devices_for_host(devices, "h-other", "10.0.0.9", "x")] == [] # bound elsewhere + assert [d.name for d in devices_for_host(devices, "h1", "192.168.1.2", None)] == ["sw"] + assert devices_for_host(devices, "h1", "", None) == [] diff --git a/tests/plugins/snmp/test_snmp_widget.py b/tests/plugins/snmp/test_snmp_widget.py index 9d8823e..63f8564 100644 --- a/tests/plugins/snmp/test_snmp_widget.py +++ b/tests/plugins/snmp/test_snmp_widget.py @@ -1,56 +1,37 @@ -"""The dashboard SNMP widget (#5255): a verdict, devices that stopped answering -first, status in words, and key readings with their units.""" +"""The dashboard SNMP widget (#5255, milestone 491): a verdict, devices not +answering first with since when, status in words, two readings in units.""" from datetime import datetime, timedelta, timezone -import pytest - -from plugins.snmp.routes import key_readings, snmp_reading from tests._templates import render_plugin_template -@pytest.mark.parametrize("label, value, says", [ - ("uptime_centisecs", 105_120_000, ("uptime", "12d 4h")), - ("battery_charge_pct", 99.6, ("battery charge", "100%")), - ("input_voltage_dV", 1203, ("input voltage", "120.3 V")), - ("if1_in_octets", 1536, ("if1 in", "1.5 KiB total")), - ("battery_status", 2, ("battery status", "2")), - ("temperature", 41.25, ("temperature", "41.25")), -]) -def test_a_reading_reads_with_its_unit(label, value, says): - assert snmp_reading(label, value) == says - - -def test_key_readings_prefer_gauges_to_counters_in_config_order(): - oids = [{"oid": "a", "label": "if1_in_octets"}, {"oid": "b", "label": "uptime_centisecs"}, - {"oid": "c", "label": "if1_out_octets"}, {"oid": "1.2.3"}] - readings = {"if1_in_octets": 10, "uptime_centisecs": 6000, "if1_out_octets": 20, "1.2.3": 7} - assert [n for n, _ in key_readings(oids, readings)] == ["uptime", "1.2.3"] - assert [n for n, _ in key_readings(oids[:1] + oids[2:3], readings)] == ["if1 in", "if1 out"] - assert key_readings(oids, {}) == [] - - -def _dev(name, reach, minutes_ago=None, key=()): - at = datetime.now(timezone.utc) - timedelta(minutes=minutes_ago) if minutes_ago is not None else None - return {"name": name, "reach": reach, "at": at, "readings": {}, "key": list(key)} +def _dev(name, reach, ok_minutes=None, key=()): + now = datetime.now(timezone.utc) + return {"id": f"id-{name}", "name": name, "reach": reach, "says": "No reply.", + "polled_at": now - timedelta(seconds=30), + "ok_at": now - timedelta(minutes=ok_minutes) if ok_minutes is not None else None, + "key": [{"name": n, "text": t} for n, t in key]} async def test_a_silent_device_leads_with_since_when(app): - devices = [_dev("ups", "stale", 14, [("battery charge", "100%")]), _dev("printer", "pending"), - _dev("switch", "up", 1, [("uptime", "12d 4h")])] + devices = [_dev("ups", "down", 14, [("Charge", "100%")]), _dev("printer", "pending"), + _dev("switch", "up", 1, [("Uptime", "12d 4h")])] async with app.test_request_context("/plugins/snmp/widget"): page = await render_plugin_template(app, "snmp", "snmp/widget.html", devices=devices, more=0, - counts={"stale": 1, "pending": 1, "up": 1}, total=3) + counts={"down": 1, "stale": 0, "pending": 1, "up": 1}, total=3) page = " ".join(page.split()) - assert "ups: no reply since" in page and ">14m ago<" in page + assert "ups isn't answering." in page and "no reply since" in page and ">14m ago<" in page assert page.count("Device") == 1 assert page.index('title="ups"') < page.index('title="printer"') < page.index('title="switch"') - assert "no data yet" in page and "uptime 12d 4h" in page - assert "detail →" not in page and "○" not in page + assert "not polled yet" in page and "Uptime 12d 4h" in page async def test_all_answering_says_so(app): async with app.test_request_context("/plugins/snmp/widget"): page = await render_plugin_template(app, "snmp", "snmp/widget.html", devices=[_dev("switch", "up", 1)], more=0, - counts={"stale": 0, "pending": 0, "up": 1}, total=1) + counts={"down": 0, "stale": 0, "pending": 0, "up": 1}, total=1) + empty = await render_plugin_template(app, "snmp", "snmp/widget.html", devices=[], more=0, + counts={}, total=0) assert "All 1 device answering." in page and "—" in page + assert "No SNMP devices yet." in empty