# plugins/snmp/scheduler.py from __future__ import annotations import asyncio import logging from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import TYPE_CHECKING from sqlalchemy import select from steward.core.scheduler import ScheduledTask if TYPE_CHECKING: from quart import Quart logger = logging.getLogger(__name__) # Counters' previous samples, for their rates (service.RateKeeper). RATES = None # How long a device's discovered table rows stand before they are walked # again: a disk swapped or a volume added shows within the hour, and a NAS is # walked 24 times a day, not 1,440. TABLES_REFRESH = timedelta(hours=1) def tables_due(device, now: datetime) -> bool: """Whether this poll walks the device's vendor tables: it reads some, it answered its last poll (a device that isn't answering is only asked its GET), and its rows are unknown, reset by a save, or an hour old.""" from .readings import tables_for return (bool(tables_for(device.presets or [])) and not device.last_error and (device.tables_at is None or now - device.tables_at >= TABLES_REFRESH)) def make_poll_task(app: "Quart") -> ScheduledTask: interval = app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60) async def poll() -> None: await _do_poll(app) return ScheduledTask( name="snmp_poll", coro_factory=poll, interval_seconds=int(interval), run_on_startup=True, ) # Polls collected but not yet stored, written one store at a time (#5995): a # stall of the storage under Postgres delays them instead of losing them. Two # hours at the default one-minute poll. HELD_POLLS = 120 # How long the device list may take to read before the last one read stands # in, so a stalled database doesn't stop the polling it is about to explain. DEVICES_READ_S = 10.0 # How long a poll waits for its own store before collecting on. STORE_WAIT_S = 20.0 # Each device as last read: plain copies, so a poll can go on without the # database, and a walk's rows found meanwhile can be kept on them. DEVICES: list | None = None @dataclass class Polled: """One device's poll, collected and waiting to be stored.""" at: datetime device_id: str name: str failed: str | None = None # the poll raised: why, in words error: str | None = None scan: object = None # a TableScan when the tables were walked answered: bool = False missing: list = field(default_factory=list) texts: dict = field(default_factory=dict) values: list = field(default_factory=list) # (metric, value) def _snapshot(device) -> SimpleNamespace: from sqlalchemy import inspect as sa_inspect return SimpleNamespace(**{a.key: getattr(device, a.key) for a in sa_inspect(type(device)).column_attrs}) async def _devices(app: "Quart") -> list: """The devices to poll: read now, or as last read when the database doesn't answer within DEVICES_READ_S.""" global DEVICES from .models import SnmpDevice async def read() -> list: async with app.db_sessionmaker() as session: return [_snapshot(d) for d in (await session.execute(select(SnmpDevice))).scalars()] try: DEVICES = await asyncio.wait_for(read(), DEVICES_READ_S) except Exception as exc: if DEVICES is None: logger.warning("SNMP devices not read (%s); nothing to poll yet", exc or type(exc).__name__) return [] logger.warning("SNMP devices not read within %.0f s (%s); polling the %d last read", DEVICES_READ_S, exc or type(exc).__name__, len(DEVICES)) return DEVICES async def _do_poll(app: "Quart", poll=None, walk=None) -> None: """Poll every device at once, hold what each answered, and store it: the device's outcome on its row, its answers in the metric tables, each at the time it was collected. A device whose vendor tables are due is walked first, so a new disk is read in the same poll. When the database doesn't take the store within STORE_WAIT_S the poll returns and the next one collects on; what is held is stored, oldest first, once it does (#5995). `poll` and `walk` stand in for poller.poll_device and poller.walk_tables in tests.""" global RATES from steward.core.holding import holding from .poller import poll_device, walk_tables from .readings import state_of, tables_for from .service import RateKeeper, credentials, readings_for poll = poll or poll_device walk = walk or walk_tables if RATES is None: RATES = RateKeeper() held = holding(app, "SNMP poll", store=lambda rounds: _store(app, rounds), limit=HELD_POLLS) devices = await _devices(app) if devices: started = datetime.now(timezone.utc) scans, asked = {}, {} async def one(d): creds = credentials(d) if tables_due(d, started): scans[d.id] = await walk(d.address, d.port, creds, tables_for(d.presets or [])) found = scans.get(d.id) asked[d.id] = readings_for(d, found.rows if found is not None and found.error is None else None) return await poll(d.address, d.port, creds, [(r.oid, r.key) for r in asked[d.id]]) results = await asyncio.gather(*(one(d) for d in devices), return_exceptions=True) now = datetime.now(timezone.utc) polls = [] for device, result in zip(devices, results): if isinstance(result, BaseException): logger.warning("SNMP poll of %s failed: %s", device.name, result) polls.append(Polled(now, device.id, device.name, failed=f"SNMP poll failed: {result}")) device.last_error = polls[-1].failed continue p = Polled(now, device.id, device.name, error=result.error, scan=scans.get(device.id), answered=bool(result.readings or result.texts), missing=list(result.missing)) if p.scan is not None and p.scan.error is None: # Kept on the copy too, so a stall doesn't walk it again every poll. device.tables, device.tables_at = p.scan.rows, now device.last_error = result.error missing = set(result.missing) for reading in asked[device.id]: value = result.readings.get(reading.key) if reading.unit == "state": # Its word is shown as answered; its level is stored, # unless Steward doesn't know the word (level None). raw = value if value is not None else result.texts.get(reading.key) if raw is None: continue p.texts[reading.key], value = state_of(reading, raw) elif value is None: if reading.key in result.texts: # a word where a number was asked for missing.add(reading.key) continue elif reading.counter: value = RATES.rate(device.id, reading, value, now) else: value = value * reading.scale + reading.offset if value is not None: p.values.append((reading.key, value)) p.missing = sorted(missing) polls.append(p) held.hold(polls) await held.flush(STORE_WAIT_S) async def _store(app: "Quart", rounds: list[list[Polled]]) -> None: """Write held polls, oldest first, in one transaction.""" from steward.core.alerts import MetricBatch from .models import SnmpDevice async with app.db_sessionmaker() as session: async with session.begin(): batch = MetricBatch() for polls in rounds: for p in polls: row = await session.get(SnmpDevice, p.device_id) if row is None: # deleted since it was polled continue row.last_polled_at = p.at if p.failed: row.last_error = p.failed continue if p.scan is not None and p.scan.error is None: row.tables, row.tables_at = p.scan.rows, p.at row.last_error = p.error if p.answered: row.last_ok_at = p.at # A failed poll keeps the words it had; one that answered says them afresh. texts = dict(row.texts or {}) if p.error else {} texts.update(p.texts) row.missing, row.texts = p.missing, texts for key, value in p.values: batch.add(source_module="snmp", resource_name=p.name, metric_name=key, value=value, recorded_at=p.at) await batch.record(session)