FabledSteward/tests/integration/test_snmp_poll.py

"""Integration: an SNMP poll writes each device's outcome to its row and its
answers to the metric tables, a counter as a rate (milestone 491).
Requires STEWARD_DATABASE_URL."""
from __future__ import annotations

import asyncio
import importlib
import os
import uuid

import pytest

pytestmark = pytest.mark.integration


@pytest.fixture
def app():
    if not os.environ.get("STEWARD_DATABASE_URL"):
        pytest.skip("needs Postgres")
    from steward.app import create_app
    return create_app(testing=False)


def _snmp(sub):
    """A submodule of the snmp plugin as the app loaded it: it is on by
    default, so its models are already mapped under the loader's name, and
    importing plugins.snmp.models would define the table a second time."""
    from steward.core.plugin_manager import get_loaded_plugin_modules
    loaded = get_loaded_plugin_modules().get("snmp")
    return importlib.import_module(f"{loaded.__name__ if loaded else 'plugins.snmp'}.{sub}")


def test_a_poll_records_why_a_device_is_silent_and_what_another_answered(app):
    from sqlalchemy import select

    from steward.models.metrics import PluginMetricLatest

    scheduler = _snmp("scheduler")
    SnmpDevice = _snmp("models").SnmpDevice
    PollResult = _snmp("poller").PollResult

    tag = uuid.uuid4().hex[:8]
    up, down = f"sw-{tag}", f"ups-{tag}"
    counts = iter([1_000.0, 61_000.0])

    async def fake_poll(address, port, creds, oids):
        if address == "10.9.9.9":
            return PollResult(error="No reply from 10.9.9.9:161 within 5 s.")
        keys = dict((k, o) for o, k in oids)
        assert {"uptime", "if1_in", "if1_status"} <= set(keys) and creds.community == "public"
        return PollResult(readings={"uptime": 8_640_000.0, "if1_status": 1.0, "if1_in": next(counts)},
                          missing=["if1_out"])

    async def go():
        from steward.core.crypto import encrypt_secret
        async with app.db_sessionmaker() as s:
            async with s.begin():
                s.add(SnmpDevice(name=up, address="10.9.9.1", presets=["system", "interfaces"],
                                 interfaces="1", community=encrypt_secret("public")))
                s.add(SnmpDevice(name=down, address="10.9.9.9", presets=["ups"],
                                 community=encrypt_secret("public")))
        scheduler.RATES = None
        await scheduler._do_poll(app, poll=fake_poll)
        first = await _latest(app, up)
        await asyncio.sleep(1.1)
        await scheduler._do_poll(app, poll=fake_poll)
        async with app.db_sessionmaker() as s:
            async with s.begin():
                rows = {d.name: d for d in (await s.execute(
                    select(SnmpDevice).where(SnmpDevice.name.in_([up, down])))).scalars()}
                for d in rows.values():
                    await s.delete(d)
        return first, await _latest(app, up), rows

    async def _latest(app, name):
        async with app.db_sessionmaker() as s:
            return {r.metric_name: r.value for r in (await s.execute(select(PluginMetricLatest).where(
                PluginMetricLatest.source_module == "snmp", PluginMetricLatest.resource_name == name))).scalars()}

    first, second, rows = asyncio.run(go())
    assert rows[up].last_error is None and rows[up].last_ok_at is not None and rows[up].missing == ["if1_out"]
    assert rows[down].last_error == "No reply from 10.9.9.9:161 within 5 s." and rows[down].last_ok_at is None
    assert rows[down].last_polled_at is not None
    assert first == {"uptime": 8_640_000.0, "if1_status": 1.0}          # a counter needs two samples
    # 60,000 bytes over the 1.1 s or more between the polls, stored as bits
    # per second: at most 436 kbit/s, and above 150 unless a poll took seconds.
    assert 150_000 < second["if1_in"] <= 436_364


def test_a_nas_is_walked_once_then_read_by_row_its_states_stored_as_levels_and_shown_in_its_words(app):
    from sqlalchemy import select

    from steward.models.metrics import PluginMetricLatest

    scheduler = _snmp("scheduler")
    SnmpDevice = _snmp("models").SnmpDevice
    poller = _snmp("poller")

    name = f"nas-{uuid.uuid4().hex[:8]}"
    walks = []

    async def fake_walk(address, port, creds, tables):
        walks.append([t.id for t in tables])
        return poller.TableScan(rows={
            "asustor_disks": [{"index": "1", "label": "Disk1", "columns": ["status", "temp"]}],
            "asustor_volumes": [{"index": "1", "label": "Volume 1", "columns": ["status", "free"]}]})

    async def fake_poll(address, port, creds, oids):
        keys = {k for _, k in oids}
        assert {"disk1_status", "disk1_temp", "volume1_status", "volume1_free"} <= keys
        return poller.PollResult(readings={"disk1_temp": 41.0, "volume1_free": 100.0},
                                 texts={"disk1_status": "Normal", "volume1_status": "Degraded"})

    async def go():
        from steward.core.crypto import encrypt_secret
        async with app.db_sessionmaker() as s:
            async with s.begin():
                s.add(SnmpDevice(name=name, address="10.9.9.5", presets=["asustor"],
                                 community=encrypt_secret("public")))
        scheduler.RATES = None
        await scheduler._do_poll(app, poll=fake_poll, walk=fake_walk)
        await scheduler._do_poll(app, poll=fake_poll, walk=fake_walk)
        async with app.db_sessionmaker() as s:
            async with s.begin():
                row = (await s.execute(select(SnmpDevice).where(SnmpDevice.name == name))).scalar_one()
                latest = {r.metric_name: r.value for r in (await s.execute(select(PluginMetricLatest).where(
                    PluginMetricLatest.source_module == "snmp", PluginMetricLatest.resource_name == name))).scalars()}
                await s.delete(row)
        return row, latest

    row, latest = asyncio.run(go())
    assert len(walks) == 1 and walks[0][:2] == ["asustor_disks", "asustor_volumes"]   # not again within the hour
    assert row.tables_at is not None and row.tables["asustor_volumes"][0]["label"] == "Volume 1"
    assert row.texts == {"disk1_status": "Normal", "volume1_status": "Degraded"} and row.last_error is None
    assert latest["disk1_status"] == 0.0 and latest["volume1_status"] == 2.0 and latest["disk1_temp"] == 41.0
    assert latest["volume1_free"] == 100.0 * 2 ** 30                   # ADM's GB, stored in bytes