feat(snmp): NAS presets for Synology and Asustor, reading every disk and volume (#5985, #5986)

A preset can now read a vendor's tables: one row per disk, volume, fan or
sensor. The scheduled poll discovers the rows by walking the device: on its
first poll, after a save, and then hourly, and only while it answers. Each row
becomes readings named by the device's own labels ("Disk 1 status"). Only the
columns a row answered are asked for.

A vendor status, whether a code (Synology) or a word (Asustor), is a state
reading. It is stored as a level (0 OK, 1 busy, 2 warning, 3 failed), so one
alert rule covers any vendor, and shown in the device's own word.

- Synology: system, power and fan status and temperature; per disk: status,
  health, temperature and bad sectors; per volume: RAID state (scrubbing,
  parity check, rebuild, degraded), free space and size.
- Asustor: per disk: status, SMART and temperature; per volume: status, free
  space and size; CPU and system temperature, free memory and fans.
  From ADM's ASUSTOR-DISK-MIB and ASUSTOR-SYSTEM-MIB.
- Linux server: plus each physical disk's 1-minute busy % (UCD-DISKIO-MIB).

Migration snmp_002_table_rows adds tables, tables_at and texts to
snmp_devices. Milestone 530.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/snmp/scheduler.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING
 
 from sqlalchemy import select
@@ -18,6 +18,20 @@ logger = logging.getLogger(__name__)
 # Counters' previous samples, for their rates (service.RateKeeper).
 RATES = None
 
+# How long a device's discovered table rows stand before they are walked
+# again: a disk swapped or a volume added shows within the hour, and a NAS is
+# walked 24 times a day, not 1,440.
+TABLES_REFRESH = timedelta(hours=1)
+
+
+def tables_due(device, now: datetime) -> bool:
+    """Whether this poll walks the device's vendor tables: it reads some, it
+    answered its last poll (a device that isn't answering is only asked its
+    GET), and its rows are unknown, reset by a save, or an hour old."""
+    from .readings import tables_for
+    return (bool(tables_for(device.presets or [])) and not device.last_error
+            and (device.tables_at is None or now - device.tables_at >= TABLES_REFRESH))
+
 
 def make_poll_task(app: "Quart") -> ScheduledTask:
     interval = app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60)
@@ -33,18 +47,21 @@ def make_poll_task(app: "Quart") -> ScheduledTask:
     )
 
 
-async def _do_poll(app: "Quart", poll=None) -> None:
+async def _do_poll(app: "Quart", poll=None, walk=None) -> None:
     """Poll every device at once, then write each one's outcome to its row
-    and its answers to the metric tables. `poll` stands in for
-    poller.poll_device in tests."""
+    and its answers to the metric tables. A device whose vendor tables are
+    due is walked first, so a new disk is read in the same poll. `poll` and
+    `walk` stand in for poller.poll_device and poller.walk_tables in tests."""
     global RATES
     from steward.core.alerts import MetricBatch
 
     from .models import SnmpDevice
-    from .poller import poll_device
+    from .poller import poll_device, walk_tables
+    from .readings import state_of, tables_for
     from .service import RateKeeper, credentials, readings_for
 
     poll = poll or poll_device
+    walk = walk or walk_tables
     if RATES is None:
         RATES = RateKeeper()
 
@@ -53,10 +70,18 @@ async def _do_poll(app: "Quart", poll=None) -> None:
     if not devices:
         return
 
-    asked = {d.id: readings_for(d) for d in devices}
-    results = await asyncio.gather(*(
-        poll(d.address, d.port, credentials(d), [(r.oid, r.key) for r in asked[d.id]])
-        for d in devices), return_exceptions=True)
+    started = datetime.now(timezone.utc)
+    scans, asked = {}, {}
+
+    async def one(d):
+        creds = credentials(d)
+        if tables_due(d, started):
+            scans[d.id] = await walk(d.address, d.port, creds, tables_for(d.presets or []))
+        found = scans.get(d.id)
+        asked[d.id] = readings_for(d, found.rows if found is not None and found.error is None else None)
+        return await poll(d.address, d.port, creds, [(r.oid, r.key) for r in asked[d.id]])
+
+    results = await asyncio.gather(*(one(d) for d in devices), return_exceptions=True)
 
     now = datetime.now(timezone.utc)
     async with app.db_sessionmaker() as session:
@@ -71,20 +96,35 @@ async def _do_poll(app: "Quart", poll=None) -> None:
                     logger.warning("SNMP poll of %s failed: %s", device.name, result)
                     row.last_error = f"SNMP poll failed: {result}"
                     continue
+                scan = scans.get(device.id)
+                if scan is not None and scan.error is None:
+                    row.tables, row.tables_at = scan.rows, now
                 row.last_error = result.error
-                row.missing = sorted(set(result.missing))
-                if result.readings:
+                missing = set(result.missing)
+                if result.readings or result.texts:
                     row.last_ok_at = now
+                texts = dict(row.texts or {}) if result.error else {}
                 for reading in asked[device.id]:
                     value = result.readings.get(reading.key)
-                    if value is None:
-                        continue
-                    if reading.counter:
-                        value = RATES.rate(device.id, reading, value, now)
-                        if value is None:
+                    if reading.unit == "state":
+                        # Its word is shown as answered; its level is stored,
+                        # unless Steward doesn't know the word (level None).
+                        raw = value if value is not None else result.texts.get(reading.key)
+                        if raw is None:
                             continue
+                        texts[reading.key], value = state_of(reading, raw)
+                    elif value is None:
+                        if reading.key in result.texts:      # a word where a number was asked for
+                            missing.add(reading.key)
+                        continue
+                    elif reading.counter:
+                        value = RATES.rate(device.id, reading, value, now)
                     else:
                         value = value * reading.scale + reading.offset
+                    if value is None:
+                        continue
                     batch.add(source_module="snmp", resource_name=device.name,
                               metric_name=reading.key, value=value, recorded_at=now)
+                row.missing = sorted(missing)
+                row.texts = texts
             await batch.record(session)