bvandeusenandClaude Opus 5.5
fcd58224de
The SNMP poll asks each device's sysName in the same GET and keeps it (snmp_003_sys_name). A syslog source told apart by name finds its own address from the SNMP device (reported or given name) or host of that name, so its lines reach the SNMP device's and host's pages again; the edit page gains Device address, which always wins (syslog_004). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
247 lines
10 KiB
Python
247 lines
10 KiB
Python
# plugins/snmp/scheduler.py
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.core.scheduler import ScheduledTask
|
|
|
|
if TYPE_CHECKING:
|
|
from quart import Quart
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Counters' previous samples, for their rates (service.RateKeeper).
|
|
RATES = None
|
|
|
|
# How long a device's discovered table rows stand before they are walked
|
|
# again: a disk swapped or a volume added shows within the hour, and a NAS is
|
|
# walked 24 times a day, not 1,440.
|
|
TABLES_REFRESH = timedelta(hours=1)
|
|
|
|
|
|
def tables_due(device, now: datetime) -> bool:
|
|
"""Whether this poll walks the device's vendor tables: it reads some, it
|
|
answered its last poll (a device that isn't answering is only asked its
|
|
GET), and its rows are unknown, reset by a save, or an hour old."""
|
|
from .readings import tables_for
|
|
return (bool(tables_for(device.presets or [])) and not device.last_error
|
|
and (device.tables_at is None or now - device.tables_at >= TABLES_REFRESH))
|
|
|
|
|
|
def make_poll_task(app: "Quart") -> ScheduledTask:
|
|
interval = app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60)
|
|
|
|
async def poll() -> None:
|
|
await _do_poll(app)
|
|
|
|
return ScheduledTask(
|
|
name="snmp_poll",
|
|
coro_factory=poll,
|
|
interval_seconds=int(interval),
|
|
run_on_startup=True,
|
|
)
|
|
|
|
|
|
# Polls collected but not yet stored, written one store at a time (#5995): a
|
|
# stall of the storage under Postgres delays them instead of losing them. Two
|
|
# hours at the default one-minute poll.
|
|
HELD_POLLS = 120
|
|
# How long the device list may take to read before the last one read stands
|
|
# in, so a stalled database doesn't stop the polling it is about to explain.
|
|
DEVICES_READ_S = 10.0
|
|
# How long a poll waits for its own store before collecting on.
|
|
STORE_WAIT_S = 20.0
|
|
|
|
# Each device as last read: plain copies, so a poll can go on without the
|
|
# database, and a walk's rows found meanwhile can be kept on them.
|
|
DEVICES: list | None = None
|
|
|
|
|
|
@dataclass
|
|
class Polled:
|
|
"""One device's poll, collected and waiting to be stored."""
|
|
at: datetime
|
|
device_id: str
|
|
name: str
|
|
failed: str | None = None # the poll raised: why, in words
|
|
error: str | None = None
|
|
scan: object = None # a TableScan when the tables were walked
|
|
answered: bool = False
|
|
missing: list = field(default_factory=list)
|
|
texts: dict = field(default_factory=dict)
|
|
values: list = field(default_factory=list) # (metric, value)
|
|
sys_name: str | None = None
|
|
|
|
|
|
def _snapshot(device) -> SimpleNamespace:
|
|
from sqlalchemy import inspect as sa_inspect
|
|
return SimpleNamespace(**{a.key: getattr(device, a.key)
|
|
for a in sa_inspect(type(device)).column_attrs})
|
|
|
|
|
|
async def _devices(app: "Quart") -> list:
|
|
"""The devices to poll: read now, or as last read when the database
|
|
doesn't answer within DEVICES_READ_S."""
|
|
global DEVICES
|
|
from .models import SnmpDevice
|
|
|
|
async def read() -> list:
|
|
async with app.db_sessionmaker() as session:
|
|
return [_snapshot(d) for d in (await session.execute(select(SnmpDevice))).scalars()]
|
|
|
|
try:
|
|
DEVICES = await asyncio.wait_for(read(), DEVICES_READ_S)
|
|
except Exception as exc:
|
|
if DEVICES is None:
|
|
logger.warning("SNMP devices not read (%s); nothing to poll yet", exc or type(exc).__name__)
|
|
return []
|
|
logger.warning("SNMP devices not read within %.0f s (%s); polling the %d last read",
|
|
DEVICES_READ_S, exc or type(exc).__name__, len(DEVICES))
|
|
return DEVICES
|
|
|
|
|
|
async def _do_poll(app: "Quart", poll=None, walk=None) -> None:
|
|
"""Poll every device at once, hold what each answered, and store it: the
|
|
device's outcome on its row, its answers in the metric tables, each at
|
|
the time it was collected. A device whose vendor tables are due is walked
|
|
first, so a new disk is read in the same poll. When the database doesn't
|
|
take the store within STORE_WAIT_S the poll returns and the next one
|
|
collects on; what is held is stored, oldest first, once it does (#5995).
|
|
`poll` and `walk` stand in for poller.poll_device and poller.walk_tables
|
|
in tests."""
|
|
global RATES
|
|
from steward.core.holding import holding
|
|
|
|
from .poller import SYS_NAME, poll_device, walk_tables
|
|
from .readings import state_of, tables_for
|
|
from .service import RateKeeper, credentials, readings_for
|
|
|
|
poll = poll or poll_device
|
|
walk = walk or walk_tables
|
|
if RATES is None:
|
|
RATES = RateKeeper()
|
|
held = holding(app, "SNMP poll", store=lambda rounds: _store(app, rounds), limit=HELD_POLLS)
|
|
|
|
devices = await _devices(app)
|
|
if devices:
|
|
started = datetime.now(timezone.utc)
|
|
scans, asked = {}, {}
|
|
|
|
async def one(d):
|
|
creds = credentials(d)
|
|
if tables_due(d, started):
|
|
scans[d.id] = await walk(d.address, d.port, creds, tables_for(d.presets or []))
|
|
found = scans.get(d.id)
|
|
asked[d.id] = readings_for(d, found.rows if found is not None and found.error is None else None)
|
|
# The device's own name rides along in the same GET.
|
|
return await poll(d.address, d.port, creds,
|
|
[(r.oid, r.key) for r in _asks(asked[d.id])] + [(SYS_NAME, SYS_NAME_KEY)])
|
|
|
|
results = await asyncio.gather(*(one(d) for d in devices), return_exceptions=True)
|
|
|
|
now = datetime.now(timezone.utc)
|
|
polls = []
|
|
for device, result in zip(devices, results):
|
|
if isinstance(result, BaseException):
|
|
logger.warning("SNMP poll of %s failed: %s", device.name, result)
|
|
polls.append(Polled(now, device.id, device.name, failed=f"SNMP poll failed: {result}"))
|
|
device.last_error = polls[-1].failed
|
|
continue
|
|
sys_name = result.texts.pop(SYS_NAME_KEY, None)
|
|
if SYS_NAME_KEY in result.missing:
|
|
result.missing.remove(SYS_NAME_KEY)
|
|
p = Polled(now, device.id, device.name, error=result.error, scan=scans.get(device.id), sys_name=sys_name,
|
|
answered=bool(result.readings or result.texts), missing=list(result.missing))
|
|
if p.scan is not None and p.scan.error is None:
|
|
# Kept on the copy too, so a stall doesn't walk it again every poll.
|
|
device.tables, device.tables_at = p.scan.rows, now
|
|
device.last_error = result.error
|
|
missing = set(result.missing)
|
|
|
|
def read(reading):
|
|
value = result.readings.get(reading.key)
|
|
if reading.unit == "state":
|
|
# Its word is shown as answered; its level is stored,
|
|
# unless Steward doesn't know the word (level None).
|
|
raw = value if value is not None else result.texts.get(reading.key)
|
|
if raw is None:
|
|
return None
|
|
p.texts[reading.key], value = state_of(reading, raw)
|
|
return value
|
|
if value is None:
|
|
if reading.key in result.texts: # a word where a number was asked for
|
|
missing.add(reading.key)
|
|
return None
|
|
if reading.counter:
|
|
return RATES.rate(device.id, reading, value, now)
|
|
return value * reading.scale + reading.offset
|
|
|
|
for reading in asked[device.id]:
|
|
if reading.inputs:
|
|
# Every input is read, so each counter keeps its last sample.
|
|
got = {i.key: read(i) for i in reading.inputs}
|
|
if any(i.key in missing for i in reading.inputs):
|
|
missing -= {i.key for i in reading.inputs}
|
|
missing.add(reading.key)
|
|
value = None if None in got.values() else reading.combine(got)
|
|
else:
|
|
value = read(reading)
|
|
if value is not None:
|
|
p.values.append((reading.key, value))
|
|
p.missing = sorted(missing)
|
|
polls.append(p)
|
|
held.hold(polls)
|
|
await held.flush(STORE_WAIT_S)
|
|
|
|
|
|
# The key the device's sysName is asked under: never a reading's.
|
|
SYS_NAME_KEY = "_sys_name"
|
|
|
|
|
|
def _asks(readings: list) -> list:
|
|
"""What the device is asked for: each reading, or the inputs of one
|
|
worked out from others."""
|
|
return [i for r in readings for i in (r.inputs or (r,))]
|
|
|
|
|
|
async def _store(app: "Quart", rounds: list[list[Polled]]) -> None:
|
|
"""Write held polls, oldest first, in one transaction."""
|
|
from steward.core.alerts import MetricBatch
|
|
|
|
from .models import SnmpDevice
|
|
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
batch = MetricBatch()
|
|
for polls in rounds:
|
|
for p in polls:
|
|
row = await session.get(SnmpDevice, p.device_id)
|
|
if row is None: # deleted since it was polled
|
|
continue
|
|
row.last_polled_at = p.at
|
|
if p.failed:
|
|
row.last_error = p.failed
|
|
continue
|
|
if p.scan is not None and p.scan.error is None:
|
|
row.tables, row.tables_at = p.scan.rows, p.at
|
|
row.last_error = p.error
|
|
if p.answered:
|
|
row.last_ok_at = p.at
|
|
if p.sys_name:
|
|
row.sys_name = p.sys_name[:255]
|
|
# A failed poll keeps the words it had; one that answered says them afresh.
|
|
texts = dict(row.texts or {}) if p.error else {}
|
|
texts.update(p.texts)
|
|
row.missing, row.texts = p.missing, texts
|
|
for key, value in p.values:
|
|
batch.add(source_module="snmp", resource_name=p.name,
|
|
metric_name=key, value=value, recorded_at=p.at)
|
|
await batch.record(session)
|