feat(syslog): a device through Docker's address is placed by its name, or a device address set on it (#6096)
The SNMP poll asks each device's sysName in the same GET and keeps it (snmp_003_sys_name). A syslog source told apart by name finds its own address from the SNMP device (reported or given name) or host of that name, so its lines reach the SNMP device's and host's pages again; the edit page gains Device address, which always wins (syslog_004). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
"""SNMP devices keep the name each reports (sysName)
|
||||
|
||||
A device whose syslog lines arrive through Docker's address is placed by the
|
||||
name its lines carry; the name the device reports over SNMP is how that name
|
||||
finds the device (#6096).
|
||||
|
||||
Revision ID: snmp_003_sys_name
|
||||
Revises: snmp_002_table_rows
|
||||
Create Date: 2026-10-19
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "snmp_003_sys_name"
|
||||
down_revision: Union[str, None] = "snmp_002_table_rows"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("snmp_devices", sa.Column("sys_name", sa.String(255), nullable=False, server_default=""))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("snmp_devices", "sys_name")
|
||||
@@ -46,3 +46,6 @@ class SnmpDevice(Base):
|
||||
tables: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
tables_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
texts: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
# The name the device reports (sysName), asked with every poll: how a
|
||||
# device whose syslog comes through Docker's address is found (#6096).
|
||||
sys_name: Mapped[str] = mapped_column(String(255), nullable=False, default="")
|
||||
|
||||
@@ -77,6 +77,7 @@ class Polled:
|
||||
missing: list = field(default_factory=list)
|
||||
texts: dict = field(default_factory=dict)
|
||||
values: list = field(default_factory=list) # (metric, value)
|
||||
sys_name: str | None = None
|
||||
|
||||
|
||||
def _snapshot(device) -> SimpleNamespace:
|
||||
@@ -118,7 +119,7 @@ async def _do_poll(app: "Quart", poll=None, walk=None) -> None:
|
||||
global RATES
|
||||
from steward.core.holding import holding
|
||||
|
||||
from .poller import poll_device, walk_tables
|
||||
from .poller import SYS_NAME, poll_device, walk_tables
|
||||
from .readings import state_of, tables_for
|
||||
from .service import RateKeeper, credentials, readings_for
|
||||
|
||||
@@ -139,7 +140,9 @@ async def _do_poll(app: "Quart", poll=None, walk=None) -> None:
|
||||
scans[d.id] = await walk(d.address, d.port, creds, tables_for(d.presets or []))
|
||||
found = scans.get(d.id)
|
||||
asked[d.id] = readings_for(d, found.rows if found is not None and found.error is None else None)
|
||||
return await poll(d.address, d.port, creds, [(r.oid, r.key) for r in _asks(asked[d.id])])
|
||||
# The device's own name rides along in the same GET.
|
||||
return await poll(d.address, d.port, creds,
|
||||
[(r.oid, r.key) for r in _asks(asked[d.id])] + [(SYS_NAME, SYS_NAME_KEY)])
|
||||
|
||||
results = await asyncio.gather(*(one(d) for d in devices), return_exceptions=True)
|
||||
|
||||
@@ -151,7 +154,10 @@ async def _do_poll(app: "Quart", poll=None, walk=None) -> None:
|
||||
polls.append(Polled(now, device.id, device.name, failed=f"SNMP poll failed: {result}"))
|
||||
device.last_error = polls[-1].failed
|
||||
continue
|
||||
p = Polled(now, device.id, device.name, error=result.error, scan=scans.get(device.id),
|
||||
sys_name = result.texts.pop(SYS_NAME_KEY, None)
|
||||
if SYS_NAME_KEY in result.missing:
|
||||
result.missing.remove(SYS_NAME_KEY)
|
||||
p = Polled(now, device.id, device.name, error=result.error, scan=scans.get(device.id), sys_name=sys_name,
|
||||
answered=bool(result.readings or result.texts), missing=list(result.missing))
|
||||
if p.scan is not None and p.scan.error is None:
|
||||
# Kept on the copy too, so a stall doesn't walk it again every poll.
|
||||
@@ -195,6 +201,10 @@ async def _do_poll(app: "Quart", poll=None, walk=None) -> None:
|
||||
await held.flush(STORE_WAIT_S)
|
||||
|
||||
|
||||
# The key the device's sysName is asked under: never a reading's.
|
||||
SYS_NAME_KEY = "_sys_name"
|
||||
|
||||
|
||||
def _asks(readings: list) -> list:
|
||||
"""What the device is asked for: each reading, or the inputs of one
|
||||
worked out from others."""
|
||||
@@ -224,6 +234,8 @@ async def _store(app: "Quart", rounds: list[list[Polled]]) -> None:
|
||||
row.last_error = p.error
|
||||
if p.answered:
|
||||
row.last_ok_at = p.at
|
||||
if p.sys_name:
|
||||
row.sys_name = p.sys_name[:255]
|
||||
# A failed poll keeps the words it had; one that answered says them afresh.
|
||||
texts = dict(row.texts or {}) if p.error else {}
|
||||
texts.update(p.texts)
|
||||
|
||||
@@ -14,8 +14,8 @@ from steward.models.users import UserRole
|
||||
|
||||
from .models import SyslogLine, SyslogSource
|
||||
from .parse import facility_name, severity_name
|
||||
from .service import (AROUND_LINES, AROUND_SEVERITY, LineFilter, filtered, lines_between, place_for, rates,
|
||||
severity_level, shown_on, source_name, sources_for)
|
||||
from .service import (AROUND_LINES, AROUND_SEVERITY, LineFilter, addresses_by_name, device_address, filtered,
|
||||
lines_between, place_for, rates, severity_level, shown_on, source_name, sources_for)
|
||||
|
||||
|
||||
async def list_syslog_sources(state: str = "") -> dict:
|
||||
@@ -28,6 +28,8 @@ async def list_syslog_sources(state: str = "") -> dict:
|
||||
A device whose lines reach Steward through Docker's own address (a port
|
||||
published on Swarm's ingress) is told apart by the name its lines carry:
|
||||
`through_docker` is true and `address` is Docker's, shared with the others.
|
||||
`device_address` is the device's own: set on the source, or found from the
|
||||
SNMP device or host its name is (else the address its lines come from).
|
||||
|
||||
Each gives its address, the host it is shown on, lines received in all
|
||||
(`lines_total`) and, for an added source, in the last hour, when it was
|
||||
@@ -43,9 +45,10 @@ async def list_syslog_sources(state: str = "") -> dict:
|
||||
sources = list((await db.execute(stmt)).scalars())
|
||||
per_hour = await rates(db, [s.id for s in sources if s.state == "accepted"], now)
|
||||
hosts = {h.id: h.name for h in (await db.execute(select(Host))).scalars()}
|
||||
named = await addresses_by_name(db) if any(s.sender for s in sources) else None
|
||||
return {"sources": [{
|
||||
"source_id": s.id, "name": source_name(s), "address": s.address, "state": s.state,
|
||||
"through_docker": bool(s.sender),
|
||||
"through_docker": bool(s.sender), "device_address": device_address(s, named),
|
||||
"host_id": s.host_id, "host_name": hosts.get(s.host_id), "calls_itself": s.hostname_seen or None,
|
||||
"lines_total": s.lines_total, "lines_last_hour": per_hour.get(s.id, 0) if s.state == "accepted" else None,
|
||||
"last_heard": iso(s.last_seen_at), "last_heard_age_s": age_s(s.last_seen_at, now),
|
||||
@@ -99,7 +102,7 @@ async def search_syslog(source: str = "", host: str = "", text: str = "", severi
|
||||
list_tool="list_syslog_sources", name=source_name)]
|
||||
if host:
|
||||
h = pick(host, list((await db.execute(select(Host))).scalars()), kind="host", list_tool="list_hosts")
|
||||
chosen = sources_for(chosen, h.id, h.address)
|
||||
chosen = sources_for(chosen, h.id, h.address, await addresses_by_name(db))
|
||||
if not chosen:
|
||||
return {"range": key, "matches": [], "returned": 0, "truncated": False,
|
||||
"note": f"No added syslog source is shown on {h.name}."}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Syslog: a source's own device address, for lines that come through Docker's
|
||||
|
||||
Revision ID: syslog_004_device_address
|
||||
Revises: syslog_003_sender
|
||||
Create Date: 2026-10-19
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "syslog_004_device_address"
|
||||
down_revision: Union[str, None] = "syslog_003_sender"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("syslog_sources",
|
||||
sa.Column("device_address", sa.String(255), nullable=False, server_default=""))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("syslog_sources", "device_address")
|
||||
@@ -30,6 +30,10 @@ class SyslogSource(Base):
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||||
address: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
sender: Mapped[str] = mapped_column(String(255), nullable=False, default="")
|
||||
# The device's own address when its lines don't come from it (through
|
||||
# Docker's): set by hand; when empty, found from an SNMP device or host
|
||||
# of the sender's name (service.device_address).
|
||||
device_address: Mapped[str] = mapped_column(String(255), nullable=False, default="")
|
||||
state: Mapped[str] = mapped_column(String(10), nullable=False, default="offered")
|
||||
name: Mapped[str] = mapped_column(String(100), nullable=False, default="")
|
||||
host_id: Mapped[str | None] = mapped_column(
|
||||
|
||||
@@ -25,7 +25,8 @@ from .patterns import (PRESETS, PRESETS_BY_KEY, SOURCE_MODULE, WINDOWS, PatternE
|
||||
resource_of)
|
||||
from .receiver import LISTENER, MAX_SENDERS, send_to, settings
|
||||
from .service import (AROUND_LINES, AROUND_SEVERITY, MARK_SEVERITY, MAX_MARKS, PANEL_LINES, PANEL_SPAN,
|
||||
SEVERITY_CHOICES, apps_of, filtered, line_filter, lines_between, place_for, rates, shown_on,
|
||||
SEVERITY_CHOICES, addresses_by_name, apps_of, device_address, filtered, line_filter,
|
||||
lines_between, place_for, rates, shown_on,
|
||||
source_name)
|
||||
|
||||
syslog_bp = Blueprint("syslog", __name__, template_folder="templates")
|
||||
@@ -53,9 +54,11 @@ def listener_view(port: int, udp: bool, tcp: bool) -> dict:
|
||||
"senders_full": False, "max_senders": MAX_SENDERS}
|
||||
|
||||
|
||||
def source_view(s: SyslogSource, hosts: dict, rate: int | None = None) -> dict:
|
||||
def source_view(s: SyslogSource, hosts: dict, rate: int | None = None, named: dict | None = None) -> dict:
|
||||
host = hosts.get(s.host_id) if s.host_id else None
|
||||
own = device_address(s, named)
|
||||
return {"id": s.id, "address": s.address, "sender": s.sender, "state": s.state, "rate": rate,
|
||||
"device_address": own if own != s.address else "",
|
||||
"docker_address": on_own_network(s.address),
|
||||
"name": s.name or s.hostname_seen or s.address,
|
||||
"hostname_seen": s.hostname_seen, "host": host,
|
||||
@@ -82,10 +85,11 @@ async def index():
|
||||
select(SyslogSource).order_by(SyslogSource.last_seen_at.desc()))).scalars())
|
||||
hosts = await _hosts(db)
|
||||
per_hour = await rates(db, [r.id for r in rows if r.state == "accepted"])
|
||||
named = await addresses_by_name(db) if any(r.sender for r in rows) else None
|
||||
by_id = {h.id: h for h in hosts}
|
||||
views = {state: [] for state in ("accepted", "offered", "ignored")}
|
||||
for row in rows:
|
||||
v = source_view(row, by_id, per_hour.get(row.id, 0) if row.state == "accepted" else None)
|
||||
v = source_view(row, by_id, per_hour.get(row.id, 0) if row.state == "accepted" else None, named)
|
||||
if row.state == "offered":
|
||||
v["suggested_host"] = suggested_host(row, hosts)
|
||||
views.setdefault(row.state, []).append(v)
|
||||
@@ -182,7 +186,8 @@ async def edit_source(source_id: str):
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
source = await _source_or_404(db, source_id)
|
||||
hosts = await _hosts(db)
|
||||
return await render_template("syslog/edit.html", source=source, hosts=hosts,
|
||||
found = device_address(source, await addresses_by_name(db)) if source.sender else source.address
|
||||
return await render_template("syslog/edit.html", source=source, hosts=hosts, found=found,
|
||||
label=source.name or source.hostname_seen or source.address)
|
||||
|
||||
|
||||
@@ -197,7 +202,10 @@ async def save_source(source_id: str):
|
||||
old_name = source.name
|
||||
source.name = await _unique_name(db, source, form.get("name") or "")
|
||||
source.host_id = await _host_id(db, form.get("host_id"))
|
||||
source.device_address = "".join((form.get("device_address") or "").split())[:255]
|
||||
detail = {"name": source.name}
|
||||
if source.device_address:
|
||||
detail["device_address"] = source.device_address
|
||||
if old_name != source.name:
|
||||
detail["old_name"] = old_name
|
||||
if old_resource != resource_of(source):
|
||||
|
||||
@@ -102,14 +102,48 @@ async def rates(db, source_ids: list[str], now: datetime | None = None) -> dict[
|
||||
return {sid: n for sid, n in rows}
|
||||
|
||||
|
||||
def sources_for(sources: list[SyslogSource], host_id: str | None, address: str | None) -> list[SyslogSource]:
|
||||
def device_address(s: SyslogSource, named: dict[str, str] | None = None) -> str:
|
||||
"""The device's own address: the one set on the source; else, for a
|
||||
device whose lines come through Docker's address (#6096), the address of
|
||||
the SNMP device or host its name is; else the address its lines come from."""
|
||||
if s.device_address:
|
||||
return s.device_address
|
||||
if s.sender and named:
|
||||
found = named.get(s.sender.lower())
|
||||
if found:
|
||||
return found
|
||||
return s.address
|
||||
|
||||
|
||||
async def addresses_by_name(db) -> dict[str, str]:
|
||||
"""{name: address} of the SNMP devices (by the name each reports, and the
|
||||
one it was given) and hosts Steward knows, names lower-cased: how a device
|
||||
behind Docker's address is placed by the name its lines carry."""
|
||||
from steward.core.plugin_manager import loaded_plugin_submodule
|
||||
from steward.models.hosts import Host
|
||||
named: dict[str, str] = {}
|
||||
for name, address in (await db.execute(select(Host.name, Host.address))).all():
|
||||
if name and address:
|
||||
named.setdefault(name.lower(), address)
|
||||
snmp = loaded_plugin_submodule("snmp", "models")
|
||||
if snmp is not None:
|
||||
rows = await db.execute(select(snmp.SnmpDevice.sys_name, snmp.SnmpDevice.name, snmp.SnmpDevice.address))
|
||||
for sys_name, name, address in rows.all():
|
||||
for n in (sys_name, name):
|
||||
if n:
|
||||
named[n.lower()] = address # a device's own name over a host's
|
||||
return named
|
||||
|
||||
|
||||
def sources_for(sources: list[SyslogSource], host_id: str | None, address: str | None,
|
||||
named: dict[str, str] | None = None) -> list[SyslogSource]:
|
||||
"""The added sources a host's or a device's page shows: those bound to
|
||||
the host, and those bound to no host whose address is the one given. A
|
||||
binding is exclusive, as for SNMP devices: a source bound elsewhere never
|
||||
matches by address."""
|
||||
the host, and those bound to no host whose device address is the one
|
||||
given. A binding is exclusive, as for SNMP devices: a source bound
|
||||
elsewhere never matches by address."""
|
||||
address = (address or "").strip().lower()
|
||||
return [s for s in sources if s.state == "accepted" and (
|
||||
s.host_id == host_id if s.host_id else bool(address) and s.address.lower() == address)]
|
||||
s.host_id == host_id if s.host_id else bool(address) and device_address(s, named).lower() == address)]
|
||||
|
||||
|
||||
# ── Lines beside the readings: an anomaly's circumstances, a chart's marks (#6002)
|
||||
@@ -135,7 +169,8 @@ async def shown_on(db, host_id: str | None, address: str | None) -> list[SyslogS
|
||||
host = await db.get(Host, host_id)
|
||||
address = host.address if host is not None else None
|
||||
accepted = list((await db.execute(select(SyslogSource).where(SyslogSource.state == "accepted"))).scalars())
|
||||
return sources_for(accepted, host_id, address)
|
||||
named = await addresses_by_name(db) if any(s.sender for s in accepted) else None
|
||||
return sources_for(accepted, host_id, address, named)
|
||||
|
||||
|
||||
async def place_for(db, source: str, resource: str) -> tuple[str | None, str | None]:
|
||||
|
||||
@@ -17,9 +17,14 @@
|
||||
<input class="input" id="syslog-name" name="name" maxlength="100" value="{{ source.name }}" placeholder="{{ source.hostname_seen or source.address }}" autocomplete="off">
|
||||
</div>
|
||||
<div class="field">
|
||||
<span class="field-label">Address</span>
|
||||
<span class="mono">{{ source.address }}</span>
|
||||
<span class="field-hint">Lines are matched to a source by the address they come from.</span>
|
||||
<span class="field-label">Lines come from</span>
|
||||
<span class="mono">{{ source.address }}{% if source.sender %} as {{ source.sender }}{% endif %}</span>
|
||||
<span class="field-hint">{% if source.sender %}Docker's address, through Swarm's ingress: this device is told apart by the name in its lines.{% else %}Lines are matched to a source by the address they come from.{% endif %}</span>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label class="field-label" for="syslog-device-address">Device address</label>
|
||||
<input class="input mono" id="syslog-device-address" name="device_address" maxlength="255" value="{{ source.device_address }}" placeholder="{{ found if found != source.address else '' }}" autocomplete="off">
|
||||
<span class="field-hint">The device's own address, which places its lines on its SNMP device's and host's pages.{% if source.sender %} {% if found != source.address %}Found as <span class="mono">{{ found }}</span> from the SNMP device or host named {{ source.sender }}; set it only to override that.{% else %}No SNMP device or host is named {{ source.sender }}, so set it here.{% endif %}{% else %} Leave it empty: it is the address the lines come from.{% endif %}</span>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label class="field-label" for="syslog-host">Show on host</label>
|
||||
|
||||
@@ -82,7 +82,7 @@
|
||||
<tbody>
|
||||
{% for s in offered %}
|
||||
<tr>
|
||||
<td>{% if s.sender %}<span class="mono">{{ s.sender }}</span><div class="field-hint">through Docker's ingress, {{ s.address }}</div>{% else %}<span class="mono">{{ s.address }}</span>{% if s.hostname_seen %}<div class="field-hint">calls itself {{ s.hostname_seen }}</div>{% endif %}{% if s.docker_address %}<div class="tone-warn" style="font-size:0.78rem;">Docker's address, not the device's</div>{% endif %}{% endif %}</td>
|
||||
<td>{% if s.sender %}<span class="mono">{{ s.sender }}</span><div class="field-hint">{% if s.device_address %}<span class="mono">{{ s.device_address }}</span>, {% endif %}through Docker's ingress</div>{% else %}<span class="mono">{{ s.address }}</span>{% if s.hostname_seen %}<div class="field-hint">calls itself {{ s.hostname_seen }}</div>{% endif %}{% if s.docker_address %}<div class="tone-warn" style="font-size:0.78rem;">Docker's address, not the device's</div>{% endif %}{% endif %}</td>
|
||||
<td>{{ "{:,}".format(s.lines_total) }}</td>
|
||||
<td>{{ s.last_seen_at|ago }}</td>
|
||||
<td class="mono" style="max-width:28rem;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ s.last_line }}"><span class="{{ tone(s.level) }}">{{ s.severity or "" }}</span> {{ s.last_line or "—" }}</td>
|
||||
@@ -121,7 +121,7 @@
|
||||
<tbody>
|
||||
{% for s in sources %}
|
||||
<tr>
|
||||
<td><a href="/plugins/syslog/sources/{{ s.id }}">{{ s.name }}</a><div class="field-hint mono">{{ s.address }}</div>{% if s.sender %}<div class="field-hint">through Docker's ingress</div>{% elif s.docker_address %}<div class="tone-warn" style="font-size:0.78rem;">Docker's address, not the device's</div>{% endif %}</td>
|
||||
<td><a href="/plugins/syslog/sources/{{ s.id }}">{{ s.name }}</a><div class="field-hint mono">{{ s.device_address or s.address }}</div>{% if s.sender %}<div class="field-hint">through Docker's ingress</div>{% elif s.docker_address %}<div class="tone-warn" style="font-size:0.78rem;">Docker's address, not the device's</div>{% endif %}</td>
|
||||
<td>{% if s.host %}<a href="/hosts/{{ s.host.id }}">{{ s.host.name }}</a>{% else %}—{% endif %}</td>
|
||||
<td class="num">{{ "{:,}".format(s.rate or 0) }}</td>
|
||||
<td>{{ s.last_seen_at|ago }}</td>
|
||||
@@ -142,7 +142,7 @@
|
||||
<tbody>
|
||||
{% for s in ignored %}
|
||||
<tr>
|
||||
<td><span class="mono">{{ s.sender or s.address }}</span>{% if s.sender %}<div class="field-hint">through Docker's ingress, {{ s.address }}</div>{% elif s.hostname_seen %}<div class="field-hint">calls itself {{ s.hostname_seen }}</div>{% endif %}</td>
|
||||
<td><span class="mono">{{ s.sender or s.address }}</span>{% if s.sender %}<div class="field-hint">{% if s.device_address %}<span class="mono">{{ s.device_address }}</span>, {% endif %}through Docker's ingress</div>{% elif s.hostname_seen %}<div class="field-hint">calls itself {{ s.hostname_seen }}</div>{% endif %}</td>
|
||||
<td>{{ "{:,}".format(s.lines_total) }}</td>
|
||||
<td>{{ s.last_seen_at|ago }}</td>
|
||||
{% if admin %}<td><form method="post" action="/plugins/syslog/sources/{{ s.id }}/offer" style="margin:0;">{{ csrf_field() }}<button type="submit" class="btn btn-sm btn-ghost">Offer again</button></form></td>{% endif %}
|
||||
|
||||
Reference in New Issue
Block a user