bvandeusenandClaude Opus 5.5
41fc8073db
On 2026-10-17 the NFS under Postgres stalled for ~15 minutes. Two things went wrong for the readings that would have explained it: - The SNMP poll stored before it polled again, so while the database hung it stopped polling. It now collects on schedule and hands each round to a Holding (steward/core/holding.py): one writer stores what is held, oldest first, each reading at the time it was collected. Bounded at two hours of polls, with a 30-minute deadline on a store; the device list read falls back to the last one read after 10 s. - "Mount not answering" (10 checks in a row) went pending on Docker02 and Docker03 and never fired: the agent sent each post's backlog with its newest, healthy sample, and that sample ended every post's run. Agent 1.22.0 queues the new sample behind the backlog, so Steward receives a host's samples in the order they were taken. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
217 lines
8.9 KiB
Python
217 lines
8.9 KiB
Python
# plugins/snmp/scheduler.py
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.core.scheduler import ScheduledTask
|
|
|
|
if TYPE_CHECKING:
|
|
from quart import Quart
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Counters' previous samples, for their rates (service.RateKeeper).
|
|
RATES = None
|
|
|
|
# How long a device's discovered table rows stand before they are walked
|
|
# again: a disk swapped or a volume added shows within the hour, and a NAS is
|
|
# walked 24 times a day, not 1,440.
|
|
TABLES_REFRESH = timedelta(hours=1)
|
|
|
|
|
|
def tables_due(device, now: datetime) -> bool:
|
|
"""Whether this poll walks the device's vendor tables: it reads some, it
|
|
answered its last poll (a device that isn't answering is only asked its
|
|
GET), and its rows are unknown, reset by a save, or an hour old."""
|
|
from .readings import tables_for
|
|
return (bool(tables_for(device.presets or [])) and not device.last_error
|
|
and (device.tables_at is None or now - device.tables_at >= TABLES_REFRESH))
|
|
|
|
|
|
def make_poll_task(app: "Quart") -> ScheduledTask:
|
|
interval = app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60)
|
|
|
|
async def poll() -> None:
|
|
await _do_poll(app)
|
|
|
|
return ScheduledTask(
|
|
name="snmp_poll",
|
|
coro_factory=poll,
|
|
interval_seconds=int(interval),
|
|
run_on_startup=True,
|
|
)
|
|
|
|
|
|
# Polls collected but not yet stored, written one store at a time (#5995): a
|
|
# stall of the storage under Postgres delays them instead of losing them. Two
|
|
# hours at the default one-minute poll.
|
|
HELD_POLLS = 120
|
|
# How long the device list may take to read before the last one read stands
|
|
# in, so a stalled database doesn't stop the polling it is about to explain.
|
|
DEVICES_READ_S = 10.0
|
|
# How long a poll waits for its own store before collecting on.
|
|
STORE_WAIT_S = 20.0
|
|
|
|
# Each device as last read: plain copies, so a poll can go on without the
|
|
# database, and a walk's rows found meanwhile can be kept on them.
|
|
DEVICES: list | None = None
|
|
|
|
|
|
@dataclass
|
|
class Polled:
|
|
"""One device's poll, collected and waiting to be stored."""
|
|
at: datetime
|
|
device_id: str
|
|
name: str
|
|
failed: str | None = None # the poll raised: why, in words
|
|
error: str | None = None
|
|
scan: object = None # a TableScan when the tables were walked
|
|
answered: bool = False
|
|
missing: list = field(default_factory=list)
|
|
texts: dict = field(default_factory=dict)
|
|
values: list = field(default_factory=list) # (metric, value)
|
|
|
|
|
|
def _snapshot(device) -> SimpleNamespace:
|
|
from sqlalchemy import inspect as sa_inspect
|
|
return SimpleNamespace(**{a.key: getattr(device, a.key)
|
|
for a in sa_inspect(type(device)).column_attrs})
|
|
|
|
|
|
async def _devices(app: "Quart") -> list:
|
|
"""The devices to poll: read now, or as last read when the database
|
|
doesn't answer within DEVICES_READ_S."""
|
|
global DEVICES
|
|
from .models import SnmpDevice
|
|
|
|
async def read() -> list:
|
|
async with app.db_sessionmaker() as session:
|
|
return [_snapshot(d) for d in (await session.execute(select(SnmpDevice))).scalars()]
|
|
|
|
try:
|
|
DEVICES = await asyncio.wait_for(read(), DEVICES_READ_S)
|
|
except Exception as exc:
|
|
if DEVICES is None:
|
|
logger.warning("SNMP devices not read (%s); nothing to poll yet", exc or type(exc).__name__)
|
|
return []
|
|
logger.warning("SNMP devices not read within %.0f s (%s); polling the %d last read",
|
|
DEVICES_READ_S, exc or type(exc).__name__, len(DEVICES))
|
|
return DEVICES
|
|
|
|
|
|
async def _do_poll(app: "Quart", poll=None, walk=None) -> None:
|
|
"""Poll every device at once, hold what each answered, and store it: the
|
|
device's outcome on its row, its answers in the metric tables, each at
|
|
the time it was collected. A device whose vendor tables are due is walked
|
|
first, so a new disk is read in the same poll. When the database doesn't
|
|
take the store within STORE_WAIT_S the poll returns and the next one
|
|
collects on; what is held is stored, oldest first, once it does (#5995).
|
|
`poll` and `walk` stand in for poller.poll_device and poller.walk_tables
|
|
in tests."""
|
|
global RATES
|
|
from steward.core.holding import holding
|
|
|
|
from .poller import poll_device, walk_tables
|
|
from .readings import state_of, tables_for
|
|
from .service import RateKeeper, credentials, readings_for
|
|
|
|
poll = poll or poll_device
|
|
walk = walk or walk_tables
|
|
if RATES is None:
|
|
RATES = RateKeeper()
|
|
held = holding(app, "SNMP poll", store=lambda rounds: _store(app, rounds), limit=HELD_POLLS)
|
|
|
|
devices = await _devices(app)
|
|
if devices:
|
|
started = datetime.now(timezone.utc)
|
|
scans, asked = {}, {}
|
|
|
|
async def one(d):
|
|
creds = credentials(d)
|
|
if tables_due(d, started):
|
|
scans[d.id] = await walk(d.address, d.port, creds, tables_for(d.presets or []))
|
|
found = scans.get(d.id)
|
|
asked[d.id] = readings_for(d, found.rows if found is not None and found.error is None else None)
|
|
return await poll(d.address, d.port, creds, [(r.oid, r.key) for r in asked[d.id]])
|
|
|
|
results = await asyncio.gather(*(one(d) for d in devices), return_exceptions=True)
|
|
|
|
now = datetime.now(timezone.utc)
|
|
polls = []
|
|
for device, result in zip(devices, results):
|
|
if isinstance(result, BaseException):
|
|
logger.warning("SNMP poll of %s failed: %s", device.name, result)
|
|
polls.append(Polled(now, device.id, device.name, failed=f"SNMP poll failed: {result}"))
|
|
device.last_error = polls[-1].failed
|
|
continue
|
|
p = Polled(now, device.id, device.name, error=result.error, scan=scans.get(device.id),
|
|
answered=bool(result.readings or result.texts), missing=list(result.missing))
|
|
if p.scan is not None and p.scan.error is None:
|
|
# Kept on the copy too, so a stall doesn't walk it again every poll.
|
|
device.tables, device.tables_at = p.scan.rows, now
|
|
device.last_error = result.error
|
|
missing = set(result.missing)
|
|
for reading in asked[device.id]:
|
|
value = result.readings.get(reading.key)
|
|
if reading.unit == "state":
|
|
# Its word is shown as answered; its level is stored,
|
|
# unless Steward doesn't know the word (level None).
|
|
raw = value if value is not None else result.texts.get(reading.key)
|
|
if raw is None:
|
|
continue
|
|
p.texts[reading.key], value = state_of(reading, raw)
|
|
elif value is None:
|
|
if reading.key in result.texts: # a word where a number was asked for
|
|
missing.add(reading.key)
|
|
continue
|
|
elif reading.counter:
|
|
value = RATES.rate(device.id, reading, value, now)
|
|
else:
|
|
value = value * reading.scale + reading.offset
|
|
if value is not None:
|
|
p.values.append((reading.key, value))
|
|
p.missing = sorted(missing)
|
|
polls.append(p)
|
|
held.hold(polls)
|
|
await held.flush(STORE_WAIT_S)
|
|
|
|
|
|
async def _store(app: "Quart", rounds: list[list[Polled]]) -> None:
|
|
"""Write held polls, oldest first, in one transaction."""
|
|
from steward.core.alerts import MetricBatch
|
|
|
|
from .models import SnmpDevice
|
|
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
batch = MetricBatch()
|
|
for polls in rounds:
|
|
for p in polls:
|
|
row = await session.get(SnmpDevice, p.device_id)
|
|
if row is None: # deleted since it was polled
|
|
continue
|
|
row.last_polled_at = p.at
|
|
if p.failed:
|
|
row.last_error = p.failed
|
|
continue
|
|
if p.scan is not None and p.scan.error is None:
|
|
row.tables, row.tables_at = p.scan.rows, p.at
|
|
row.last_error = p.error
|
|
if p.answered:
|
|
row.last_ok_at = p.at
|
|
# A failed poll keeps the words it had; one that answered says them afresh.
|
|
texts = dict(row.texts or {}) if p.error else {}
|
|
texts.update(p.texts)
|
|
row.missing, row.texts = p.missing, texts
|
|
for key, value in p.values:
|
|
batch.add(source_module="snmp", resource_name=p.name,
|
|
metric_name=key, value=value, recorded_at=p.at)
|
|
await batch.record(session)
|