diff --git a/docs/plugins/syslog.md b/docs/plugins/syslog.md index 1d32777..6beb1b8 100644 --- a/docs/plugins/syslog.md +++ b/docs/plugins/syslog.md @@ -54,9 +54,15 @@ ports: Through ingress every line arrives from Docker's own address on the ingress network, so the address no longer tells the devices apart. For such an address Steward keys the source by the hostname each line carries instead (#6096): each -device is its own source, linked to a host by name. A device must put its -hostname in its lines (NASes and routers do); lines with none at Docker's -address all fold into one source, and the page says so. +device is its own source. A device must put its hostname in its lines (NASes +and routers do); lines with none at Docker's address all fold into one source, +and the page says so. + +Such a source's own address, which places its lines on its SNMP device's and +host's pages (chart marks, the lines around an anomaly), is found by name: the +SNMP device whose reported name (sysName, asked with every poll) or given name +matches, else the host of that name. Where nothing matches, set **Device +address** on the source's edit page; a value set there always wins. **Host mode** keeps each device's own address, but only the node running the task answers, so devices must be pointed at that node: diff --git a/plugins/snmp/migrations/versions/snmp_003_sys_name.py b/plugins/snmp/migrations/versions/snmp_003_sys_name.py new file mode 100644 index 0000000..4755f0a --- /dev/null +++ b/plugins/snmp/migrations/versions/snmp_003_sys_name.py @@ -0,0 +1,27 @@ +"""SNMP devices keep the name each reports (sysName) + +A device whose syslog lines arrive through Docker's address is placed by the +name its lines carry; the name the device reports over SNMP is how that name +finds the device (#6096). + +Revision ID: snmp_003_sys_name +Revises: snmp_002_table_rows +Create Date: 2026-10-19 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "snmp_003_sys_name" +down_revision: Union[str, None] = "snmp_002_table_rows" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("snmp_devices", sa.Column("sys_name", sa.String(255), nullable=False, server_default="")) + + +def downgrade() -> None: + op.drop_column("snmp_devices", "sys_name") diff --git a/plugins/snmp/models.py b/plugins/snmp/models.py index 3da6895..6d61eb6 100644 --- a/plugins/snmp/models.py +++ b/plugins/snmp/models.py @@ -46,3 +46,6 @@ class SnmpDevice(Base): tables: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) tables_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) texts: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + # The name the device reports (sysName), asked with every poll: how a + # device whose syslog comes through Docker's address is found (#6096). + sys_name: Mapped[str] = mapped_column(String(255), nullable=False, default="") diff --git a/plugins/snmp/scheduler.py b/plugins/snmp/scheduler.py index 15de0ec..b84d0ef 100644 --- a/plugins/snmp/scheduler.py +++ b/plugins/snmp/scheduler.py @@ -77,6 +77,7 @@ class Polled: missing: list = field(default_factory=list) texts: dict = field(default_factory=dict) values: list = field(default_factory=list) # (metric, value) + sys_name: str | None = None def _snapshot(device) -> SimpleNamespace: @@ -118,7 +119,7 @@ async def _do_poll(app: "Quart", poll=None, walk=None) -> None: global RATES from steward.core.holding import holding - from .poller import poll_device, walk_tables + from .poller import SYS_NAME, poll_device, walk_tables from .readings import state_of, tables_for from .service import RateKeeper, credentials, readings_for @@ -139,7 +140,9 @@ async def _do_poll(app: "Quart", poll=None, walk=None) -> None: scans[d.id] = await walk(d.address, d.port, creds, tables_for(d.presets or [])) found = scans.get(d.id) asked[d.id] = readings_for(d, found.rows if found is not None and found.error is None else None) - return await poll(d.address, d.port, creds, [(r.oid, r.key) for r in _asks(asked[d.id])]) + # The device's own name rides along in the same GET. + return await poll(d.address, d.port, creds, + [(r.oid, r.key) for r in _asks(asked[d.id])] + [(SYS_NAME, SYS_NAME_KEY)]) results = await asyncio.gather(*(one(d) for d in devices), return_exceptions=True) @@ -151,7 +154,10 @@ async def _do_poll(app: "Quart", poll=None, walk=None) -> None: polls.append(Polled(now, device.id, device.name, failed=f"SNMP poll failed: {result}")) device.last_error = polls[-1].failed continue - p = Polled(now, device.id, device.name, error=result.error, scan=scans.get(device.id), + sys_name = result.texts.pop(SYS_NAME_KEY, None) + if SYS_NAME_KEY in result.missing: + result.missing.remove(SYS_NAME_KEY) + p = Polled(now, device.id, device.name, error=result.error, scan=scans.get(device.id), sys_name=sys_name, answered=bool(result.readings or result.texts), missing=list(result.missing)) if p.scan is not None and p.scan.error is None: # Kept on the copy too, so a stall doesn't walk it again every poll. @@ -195,6 +201,10 @@ async def _do_poll(app: "Quart", poll=None, walk=None) -> None: await held.flush(STORE_WAIT_S) +# The key the device's sysName is asked under: never a reading's. +SYS_NAME_KEY = "_sys_name" + + def _asks(readings: list) -> list: """What the device is asked for: each reading, or the inputs of one worked out from others.""" @@ -224,6 +234,8 @@ async def _store(app: "Quart", rounds: list[list[Polled]]) -> None: row.last_error = p.error if p.answered: row.last_ok_at = p.at + if p.sys_name: + row.sys_name = p.sys_name[:255] # A failed poll keeps the words it had; one that answered says them afresh. texts = dict(row.texts or {}) if p.error else {} texts.update(p.texts) diff --git a/plugins/syslog/mcp_tools.py b/plugins/syslog/mcp_tools.py index 505bd71..0131598 100644 --- a/plugins/syslog/mcp_tools.py +++ b/plugins/syslog/mcp_tools.py @@ -14,8 +14,8 @@ from steward.models.users import UserRole from .models import SyslogLine, SyslogSource from .parse import facility_name, severity_name -from .service import (AROUND_LINES, AROUND_SEVERITY, LineFilter, filtered, lines_between, place_for, rates, - severity_level, shown_on, source_name, sources_for) +from .service import (AROUND_LINES, AROUND_SEVERITY, LineFilter, addresses_by_name, device_address, filtered, + lines_between, place_for, rates, severity_level, shown_on, source_name, sources_for) async def list_syslog_sources(state: str = "") -> dict: @@ -28,6 +28,8 @@ async def list_syslog_sources(state: str = "") -> dict: A device whose lines reach Steward through Docker's own address (a port published on Swarm's ingress) is told apart by the name its lines carry: `through_docker` is true and `address` is Docker's, shared with the others. + `device_address` is the device's own: set on the source, or found from the + SNMP device or host its name is (else the address its lines come from). Each gives its address, the host it is shown on, lines received in all (`lines_total`) and, for an added source, in the last hour, when it was @@ -43,9 +45,10 @@ async def list_syslog_sources(state: str = "") -> dict: sources = list((await db.execute(stmt)).scalars()) per_hour = await rates(db, [s.id for s in sources if s.state == "accepted"], now) hosts = {h.id: h.name for h in (await db.execute(select(Host))).scalars()} + named = await addresses_by_name(db) if any(s.sender for s in sources) else None return {"sources": [{ "source_id": s.id, "name": source_name(s), "address": s.address, "state": s.state, - "through_docker": bool(s.sender), + "through_docker": bool(s.sender), "device_address": device_address(s, named), "host_id": s.host_id, "host_name": hosts.get(s.host_id), "calls_itself": s.hostname_seen or None, "lines_total": s.lines_total, "lines_last_hour": per_hour.get(s.id, 0) if s.state == "accepted" else None, "last_heard": iso(s.last_seen_at), "last_heard_age_s": age_s(s.last_seen_at, now), @@ -99,7 +102,7 @@ async def search_syslog(source: str = "", host: str = "", text: str = "", severi list_tool="list_syslog_sources", name=source_name)] if host: h = pick(host, list((await db.execute(select(Host))).scalars()), kind="host", list_tool="list_hosts") - chosen = sources_for(chosen, h.id, h.address) + chosen = sources_for(chosen, h.id, h.address, await addresses_by_name(db)) if not chosen: return {"range": key, "matches": [], "returned": 0, "truncated": False, "note": f"No added syslog source is shown on {h.name}."} diff --git a/plugins/syslog/migrations/versions/syslog_004_device_address.py b/plugins/syslog/migrations/versions/syslog_004_device_address.py new file mode 100644 index 0000000..c767810 --- /dev/null +++ b/plugins/syslog/migrations/versions/syslog_004_device_address.py @@ -0,0 +1,24 @@ +"""Syslog: a source's own device address, for lines that come through Docker's + +Revision ID: syslog_004_device_address +Revises: syslog_003_sender +Create Date: 2026-10-19 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "syslog_004_device_address" +down_revision: Union[str, None] = "syslog_003_sender" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("syslog_sources", + sa.Column("device_address", sa.String(255), nullable=False, server_default="")) + + +def downgrade() -> None: + op.drop_column("syslog_sources", "device_address") diff --git a/plugins/syslog/models.py b/plugins/syslog/models.py index e078ef6..d54858b 100644 --- a/plugins/syslog/models.py +++ b/plugins/syslog/models.py @@ -30,6 +30,10 @@ class SyslogSource(Base): id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) address: Mapped[str] = mapped_column(String(64), nullable=False) sender: Mapped[str] = mapped_column(String(255), nullable=False, default="") + # The device's own address when its lines don't come from it (through + # Docker's): set by hand; when empty, found from an SNMP device or host + # of the sender's name (service.device_address). + device_address: Mapped[str] = mapped_column(String(255), nullable=False, default="") state: Mapped[str] = mapped_column(String(10), nullable=False, default="offered") name: Mapped[str] = mapped_column(String(100), nullable=False, default="") host_id: Mapped[str | None] = mapped_column( diff --git a/plugins/syslog/routes.py b/plugins/syslog/routes.py index ddfc97e..032095e 100644 --- a/plugins/syslog/routes.py +++ b/plugins/syslog/routes.py @@ -25,7 +25,8 @@ from .patterns import (PRESETS, PRESETS_BY_KEY, SOURCE_MODULE, WINDOWS, PatternE resource_of) from .receiver import LISTENER, MAX_SENDERS, send_to, settings from .service import (AROUND_LINES, AROUND_SEVERITY, MARK_SEVERITY, MAX_MARKS, PANEL_LINES, PANEL_SPAN, - SEVERITY_CHOICES, apps_of, filtered, line_filter, lines_between, place_for, rates, shown_on, + SEVERITY_CHOICES, addresses_by_name, apps_of, device_address, filtered, line_filter, + lines_between, place_for, rates, shown_on, source_name) syslog_bp = Blueprint("syslog", __name__, template_folder="templates") @@ -53,9 +54,11 @@ def listener_view(port: int, udp: bool, tcp: bool) -> dict: "senders_full": False, "max_senders": MAX_SENDERS} -def source_view(s: SyslogSource, hosts: dict, rate: int | None = None) -> dict: +def source_view(s: SyslogSource, hosts: dict, rate: int | None = None, named: dict | None = None) -> dict: host = hosts.get(s.host_id) if s.host_id else None + own = device_address(s, named) return {"id": s.id, "address": s.address, "sender": s.sender, "state": s.state, "rate": rate, + "device_address": own if own != s.address else "", "docker_address": on_own_network(s.address), "name": s.name or s.hostname_seen or s.address, "hostname_seen": s.hostname_seen, "host": host, @@ -82,10 +85,11 @@ async def index(): select(SyslogSource).order_by(SyslogSource.last_seen_at.desc()))).scalars()) hosts = await _hosts(db) per_hour = await rates(db, [r.id for r in rows if r.state == "accepted"]) + named = await addresses_by_name(db) if any(r.sender for r in rows) else None by_id = {h.id: h for h in hosts} views = {state: [] for state in ("accepted", "offered", "ignored")} for row in rows: - v = source_view(row, by_id, per_hour.get(row.id, 0) if row.state == "accepted" else None) + v = source_view(row, by_id, per_hour.get(row.id, 0) if row.state == "accepted" else None, named) if row.state == "offered": v["suggested_host"] = suggested_host(row, hosts) views.setdefault(row.state, []).append(v) @@ -182,7 +186,8 @@ async def edit_source(source_id: str): async with current_app.db_sessionmaker() as db: source = await _source_or_404(db, source_id) hosts = await _hosts(db) - return await render_template("syslog/edit.html", source=source, hosts=hosts, + found = device_address(source, await addresses_by_name(db)) if source.sender else source.address + return await render_template("syslog/edit.html", source=source, hosts=hosts, found=found, label=source.name or source.hostname_seen or source.address) @@ -197,7 +202,10 @@ async def save_source(source_id: str): old_name = source.name source.name = await _unique_name(db, source, form.get("name") or "") source.host_id = await _host_id(db, form.get("host_id")) + source.device_address = "".join((form.get("device_address") or "").split())[:255] detail = {"name": source.name} + if source.device_address: + detail["device_address"] = source.device_address if old_name != source.name: detail["old_name"] = old_name if old_resource != resource_of(source): diff --git a/plugins/syslog/service.py b/plugins/syslog/service.py index 92bf1f1..369b11e 100644 --- a/plugins/syslog/service.py +++ b/plugins/syslog/service.py @@ -102,14 +102,48 @@ async def rates(db, source_ids: list[str], now: datetime | None = None) -> dict[ return {sid: n for sid, n in rows} -def sources_for(sources: list[SyslogSource], host_id: str | None, address: str | None) -> list[SyslogSource]: +def device_address(s: SyslogSource, named: dict[str, str] | None = None) -> str: + """The device's own address: the one set on the source; else, for a + device whose lines come through Docker's address (#6096), the address of + the SNMP device or host its name is; else the address its lines come from.""" + if s.device_address: + return s.device_address + if s.sender and named: + found = named.get(s.sender.lower()) + if found: + return found + return s.address + + +async def addresses_by_name(db) -> dict[str, str]: + """{name: address} of the SNMP devices (by the name each reports, and the + one it was given) and hosts Steward knows, names lower-cased: how a device + behind Docker's address is placed by the name its lines carry.""" + from steward.core.plugin_manager import loaded_plugin_submodule + from steward.models.hosts import Host + named: dict[str, str] = {} + for name, address in (await db.execute(select(Host.name, Host.address))).all(): + if name and address: + named.setdefault(name.lower(), address) + snmp = loaded_plugin_submodule("snmp", "models") + if snmp is not None: + rows = await db.execute(select(snmp.SnmpDevice.sys_name, snmp.SnmpDevice.name, snmp.SnmpDevice.address)) + for sys_name, name, address in rows.all(): + for n in (sys_name, name): + if n: + named[n.lower()] = address # a device's own name over a host's + return named + + +def sources_for(sources: list[SyslogSource], host_id: str | None, address: str | None, + named: dict[str, str] | None = None) -> list[SyslogSource]: """The added sources a host's or a device's page shows: those bound to - the host, and those bound to no host whose address is the one given. A - binding is exclusive, as for SNMP devices: a source bound elsewhere never - matches by address.""" + the host, and those bound to no host whose device address is the one + given. A binding is exclusive, as for SNMP devices: a source bound + elsewhere never matches by address.""" address = (address or "").strip().lower() return [s for s in sources if s.state == "accepted" and ( - s.host_id == host_id if s.host_id else bool(address) and s.address.lower() == address)] + s.host_id == host_id if s.host_id else bool(address) and device_address(s, named).lower() == address)] # ── Lines beside the readings: an anomaly's circumstances, a chart's marks (#6002) @@ -135,7 +169,8 @@ async def shown_on(db, host_id: str | None, address: str | None) -> list[SyslogS host = await db.get(Host, host_id) address = host.address if host is not None else None accepted = list((await db.execute(select(SyslogSource).where(SyslogSource.state == "accepted"))).scalars()) - return sources_for(accepted, host_id, address) + named = await addresses_by_name(db) if any(s.sender for s in accepted) else None + return sources_for(accepted, host_id, address, named) async def place_for(db, source: str, resource: str) -> tuple[str | None, str | None]: diff --git a/plugins/syslog/templates/syslog/edit.html b/plugins/syslog/templates/syslog/edit.html index 11fc22b..879ba56 100644 --- a/plugins/syslog/templates/syslog/edit.html +++ b/plugins/syslog/templates/syslog/edit.html @@ -17,9 +17,14 @@