bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: monitors, UniFi and Traefik keep collecting through a storage stall (#5995)
CI / lint (push) Successful in 3s
Details
CI / integration (push) Successful in 1m38s
Details
CI / unit (push) Successful in 1m39s
Details
CI / publish (push) Successful in 1m11s
Details

Browse Source 
The same Holding as the SNMP poll, for the rest of Steward's own
collectors:

- Monitors: the list is read within 10 s or the last one read stands in;
  a start the database doesn't take is kept in memory, so a monitor isn't
  checked twice; each result is held and stored at the time it was
  checked. A stall shows on the monitors as the checks it saw, not a gap.
- UniFi and Traefik: each poll's writes are held (hold_write) and run
  oldest first in one transaction; every reading is stamped with its scrape.
- A held round the database refuses (as opposed to can't be reached) is
  stored alone and let go, so it can't hold up every round after it.

The Prometheus scrape is left as it is: its one target, Fabled Archives,
keeps its own database on the same NFS.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-17 09:41:14 -04:00
co-authored by Claude Opus 5.5
parent 41fc8073db
commit 2df8431de9
6 changed files with 557 additions and 388 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/traefik/scheduler.py
+73 -70
View File
@@ -75,6 +75,7 @@ async def _do_scrape(app: "Quart") -> None:
fetch_router_domains, fetch_router_domains,
) )
from steward.core.alerts import MetricBatch from steward.core.alerts import MetricBatch
from steward.core.holding import hold_write
metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"] metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"]
@@ -106,81 +107,83 @@ async def _do_scrape(app: "Quart") -> None:
scraped_at = datetime.now(timezone.utc) scraped_at = datetime.now(timezone.utc)
async with app.db_sessionmaker() as session: async def write(session) -> None:
async with session.begin(): batch = MetricBatch()
batch = MetricBatch() # ── Per-service metrics ───────────────────────────────────────────
# ── Per-service metrics ─────────────────────────────────────────── for router_name, metrics in router_metrics.items():
for router_name, metrics in router_metrics.items(): row = TraefikMetric(
row = TraefikMetric( router_name=router_name,
router_name=router_name, scraped_at=scraped_at,
scraped_at=scraped_at, request_rate=metrics["request_rate"],
request_rate=metrics["request_rate"], error_rate_4xx_pct=metrics["error_rate_4xx_pct"],
error_rate_4xx_pct=metrics["error_rate_4xx_pct"], error_rate_5xx_pct=metrics["error_rate_5xx_pct"],
error_rate_5xx_pct=metrics["error_rate_5xx_pct"], latency_p50_ms=metrics["latency_p50_ms"],
latency_p50_ms=metrics["latency_p50_ms"], latency_p95_ms=metrics["latency_p95_ms"],
latency_p95_ms=metrics["latency_p95_ms"], latency_p99_ms=metrics["latency_p99_ms"],
latency_p99_ms=metrics["latency_p99_ms"], response_bytes_rate=metrics["response_bytes_rate"],
response_bytes_rate=metrics["response_bytes_rate"], )
) session.add(row)
session.add(row)
for metric_name, value in [ for metric_name, value in [
("request_rate", metrics["request_rate"]), ("request_rate", metrics["request_rate"]),
("error_rate_4xx_pct", metrics["error_rate_4xx_pct"]), ("error_rate_4xx_pct", metrics["error_rate_4xx_pct"]),
("error_rate_5xx_pct", metrics["error_rate_5xx_pct"]), ("error_rate_5xx_pct", metrics["error_rate_5xx_pct"]),
("latency_p50_ms", metrics["latency_p50_ms"]), ("latency_p50_ms", metrics["latency_p50_ms"]),
("latency_p95_ms", metrics["latency_p95_ms"]), ("latency_p95_ms", metrics["latency_p95_ms"]),
("latency_p99_ms", metrics["latency_p99_ms"]), ("latency_p99_ms", metrics["latency_p99_ms"]),
("response_bytes_rate", metrics["response_bytes_rate"]), ("response_bytes_rate", metrics["response_bytes_rate"]),
]: ]:
if value is None: # no responses since the last scrape: no latency if value is None: # no responses since the last scrape: no latency
continue continue
# Stamped with the scrape, so a router's latest figures # Stamped with the scrape, so a router's latest figures
# (routes.latest_per_router) all carry the same moment. # (routes.latest_per_router) all carry the same moment.
batch.add(
source_module="traefik",
resource_name=router_name,
metric_name=metric_name,
value=value,
recorded_at=scraped_at,
)
# ── 4xx and 5xx responses by code (#5711) ─────────────────────────
for router_name, codes in status_codes.items():
for code, count in codes.items():
session.add(TraefikStatusCode(router_name=router_name, scraped_at=scraped_at,
code=code, count=count))
# ── Global stats ──────────────────────────────────────────────────
session.add(TraefikGlobalStat(scraped_at=scraped_at, **global_stats))
# ── TLS certs (upsert by serial) ──────────────────────────────────
for cert in cert_list:
existing = await session.get(TraefikCert, cert["serial"])
if existing:
existing.cn = cert["cn"]
existing.sans = cert["sans"]
existing.not_after = cert["not_after"]
existing.scraped_at = scraped_at
else:
session.add(TraefikCert(
serial=cert["serial"],
cn=cert["cn"],
sans=cert["sans"],
not_after=cert["not_after"],
scraped_at=scraped_at,
))
# Emit cert_expiry_days for alert rules
days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0
batch.add( batch.add(
source_module="traefik", source_module="traefik",
resource_name=cert["cn"] or cert["serial"], resource_name=router_name,
metric_name="cert_expiry_days", metric_name=metric_name,
value=max(0.0, days_remaining), value=value,
recorded_at=scraped_at,
) )
await batch.record(session) # ── 4xx and 5xx responses by code (#5711) ─────────────────────────
for router_name, codes in status_codes.items():
for code, count in codes.items():
session.add(TraefikStatusCode(router_name=router_name, scraped_at=scraped_at,
code=code, count=count))
# ── Global stats ──────────────────────────────────────────────────
session.add(TraefikGlobalStat(scraped_at=scraped_at, **global_stats))
# ── TLS certs (upsert by serial) ──────────────────────────────────
for cert in cert_list:
existing = await session.get(TraefikCert, cert["serial"])
if existing:
existing.cn = cert["cn"]
existing.sans = cert["sans"]
existing.not_after = cert["not_after"]
existing.scraped_at = scraped_at
else:
session.add(TraefikCert(
serial=cert["serial"],
cn=cert["cn"],
sans=cert["sans"],
not_after=cert["not_after"],
scraped_at=scraped_at,
))
# Emit cert_expiry_days for alert rules
days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0
batch.add(
source_module="traefik",
resource_name=cert["cn"] or cert["serial"],
metric_name="cert_expiry_days",
value=max(0.0, days_remaining),
recorded_at=scraped_at,
)
await batch.record(session)
await hold_write(app, "Traefik scrape", write)
logger.debug( logger.debug(
"Traefik scrape: %d router(s), %d cert(s)", "Traefik scrape: %d router(s), %d cert(s)",
plugins/unifi/scheduler.py
+272 -266
View File
@@ -5,6 +5,7 @@ import logging
from datetime import datetime, timezone from datetime import datetime, timezone
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from steward.core.holding import hold_write
from steward.core.scheduler import ScheduledTask from steward.core.scheduler import ScheduledTask
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -130,68 +131,72 @@ async def _do_poll(app: "Quart") -> None:
top_clients_json=json.dumps(top_clients_data), top_clients_json=json.dumps(top_clients_data),
) )
async with app.db_sessionmaker() as session: async def write(session) -> None:
async with session.begin(): session.add(wan_stat)
session.add(wan_stat) session.add(client_snapshot)
session.add(client_snapshot)
# ── Devices (upsert by MAC) ─────────────────────────────────────── # ── Devices (upsert by MAC) ───────────────────────────────────────
for d in devices: for d in devices:
mac = d.get("mac", "") mac = d.get("mac", "")
if not mac: if not mac:
continue continue
last_seen_ts = d.get("last_seen") last_seen_ts = d.get("last_seen")
last_seen_dt = ( last_seen_dt = (
datetime.fromtimestamp(last_seen_ts, tz=timezone.utc) datetime.fromtimestamp(last_seen_ts, tz=timezone.utc)
if last_seen_ts else None if last_seen_ts else None
) )
existing = await session.get(UnifiDevice, mac) existing = await session.get(UnifiDevice, mac)
if existing: if existing:
existing.name = d.get("name") existing.name = d.get("name")
existing.model = d.get("model") existing.model = d.get("model")
existing.device_type = d.get("type") existing.device_type = d.get("type")
existing.ip = d.get("ip") existing.ip = d.get("ip")
existing.state = d.get("state", 0) existing.state = d.get("state", 0)
existing.uptime_seconds = d.get("uptime") existing.uptime_seconds = d.get("uptime")
existing.last_seen = last_seen_dt existing.last_seen = last_seen_dt
existing.version = d.get("version") existing.version = d.get("version")
existing.scraped_at = scraped_at existing.scraped_at = scraped_at
else: else:
session.add(UnifiDevice( session.add(UnifiDevice(
mac=mac, mac=mac,
name=d.get("name"), name=d.get("name"),
model=d.get("model"), model=d.get("model"),
device_type=d.get("type"), device_type=d.get("type"),
ip=d.get("ip"), ip=d.get("ip"),
state=d.get("state", 0), state=d.get("state", 0),
uptime_seconds=d.get("uptime"), uptime_seconds=d.get("uptime"),
last_seen=last_seen_dt, last_seen=last_seen_dt,
version=d.get("version"), version=d.get("version"),
scraped_at=scraped_at, scraped_at=scraped_at,
)) ))
# ── Alert metrics ───────────────────────────────────────────────── # ── Alert metrics ─────────────────────────────────────────────────
batch = MetricBatch() batch = MetricBatch()
if wan_stat.latency_ms is not None: if wan_stat.latency_ms is not None:
batch.add(
source_module="unifi",
resource_name="wan",
metric_name="latency_ms",
value=wan_stat.latency_ms,
)
batch.add( batch.add(
source_module="unifi", source_module="unifi",
resource_name="wan", resource_name="wan",
metric_name="is_up", metric_name="latency_ms",
value=1.0 if wan_stat.is_up else 0.0, value=wan_stat.latency_ms,
recorded_at=scraped_at,
) )
batch.add( batch.add(
source_module="unifi", source_module="unifi",
resource_name="clients", resource_name="wan",
metric_name="total_clients", metric_name="is_up",
value=float(len(clients)), value=1.0 if wan_stat.is_up else 0.0,
) recorded_at=scraped_at,
await batch.record(session) )
batch.add(
source_module="unifi",
resource_name="clients",
metric_name="total_clients",
value=float(len(clients)),
recorded_at=scraped_at,
)
await batch.record(session)
await hold_write(app, "UniFi poll", write)
logger.debug( logger.debug(
"UniFi poll: wan=%s, clients=%d, devices=%d", "UniFi poll: wan=%s, clients=%d, devices=%d",
@@ -310,227 +315,228 @@ async def _do_expanded(app: "Quart", scraped_at: datetime) -> None:
top_dpi_clients = sorted(client_dpi.values(), key=lambda x: x["bytes"], reverse=True)[:10] top_dpi_clients = sorted(client_dpi.values(), key=lambda x: x["bytes"], reverse=True)[:10]
# ── Write to DB ─────────────────────────────────────────────────────────── # ── Write to DB ───────────────────────────────────────────────────────────
async with app.db_sessionmaker() as session: async def write(session) -> None:
async with session.begin(): # WLAN stats
# WLAN stats for row in wlan_rows:
for row in wlan_rows: session.add(row)
session.add(row)
# DPI snapshot (only if we got data) # DPI snapshot (only if we got data)
if cat_summary or top_dpi_clients: if cat_summary or top_dpi_clients:
session.add(UnifiDpiSnapshot( session.add(UnifiDpiSnapshot(
scraped_at=scraped_at, scraped_at=scraped_at,
by_category_json=json.dumps(cat_summary), by_category_json=json.dumps(cat_summary),
by_client_json=json.dumps(top_dpi_clients), by_client_json=json.dumps(top_dpi_clients),
))
# Events (upsert by _id)
_category_map = {
"wlan": "wlan", "wireless": "wlan",
"wired": "wired",
"ugw": "wan", "wan": "wan",
"ids": "ids",
"system": "system",
"ap": "ap",
}
for ev in events:
uid = ev.get("_id", "")
if not uid:
continue
ts_raw = ev.get("datetime") or ev.get("time")
try:
if isinstance(ts_raw, (int, float)):
event_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
else:
event_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
except Exception:
event_time = scraped_at
subsystem = ev.get("subsystem", "system")
category = _category_map.get(subsystem, subsystem[:16])
details = {k: v for k, v in ev.items()
if k not in ("_id", "datetime", "time", "key", "subsystem", "msg", "user", "src_ip")}
existing = await session.get(UnifiEvent, uid)
if not existing:
session.add(UnifiEvent(
unifi_id=uid,
event_time=event_time,
event_key=ev.get("key", ""),
category=category,
message=ev.get("msg", ""),
source_mac=ev.get("user") or ev.get("src_mac"),
source_ip=ev.get("src_ip"),
details_json=json.dumps(details),
)) ))
# Events (upsert by _id) # Alarms (upsert)
_category_map = { for al in alarms:
"wlan": "wlan", "wireless": "wlan", uid = al.get("_id", "")
"wired": "wired", if not uid:
"ugw": "wan", "wan": "wan", continue
"ids": "ids", ts_raw = al.get("datetime") or al.get("time")
"system": "system", try:
"ap": "ap", if isinstance(ts_raw, (int, float)):
} alarm_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
for ev in events: else:
uid = ev.get("_id", "") alarm_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
if not uid: except Exception:
continue alarm_time = scraped_at
ts_raw = ev.get("datetime") or ev.get("time") existing = await session.get(UnifiAlarm, uid)
if existing:
existing.archived = al.get("archived", False)
existing.scraped_at = scraped_at
else:
session.add(UnifiAlarm(
unifi_id=uid,
alarm_time=alarm_time,
alarm_key=al.get("key", ""),
message=al.get("msg", ""),
archived=al.get("archived", False),
scraped_at=scraped_at,
))
# Known clients (upsert by MAC)
for kc in known_clients:
mac = kc.get("mac", "")
if not mac:
continue
def _ts(val):
if not val:
return None
try: try:
if isinstance(ts_raw, (int, float)): if isinstance(val, (int, float)):
event_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc) return datetime.fromtimestamp(val, tz=timezone.utc)
else: return datetime.fromisoformat(str(val).replace("Z", "+00:00"))
event_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
except Exception: except Exception:
event_time = scraped_at return None
subsystem = ev.get("subsystem", "system") existing = await session.get(UnifiKnownClient, mac)
category = _category_map.get(subsystem, subsystem[:16]) if existing:
details = {k: v for k, v in ev.items() existing.hostname = kc.get("hostname")
if k not in ("_id", "datetime", "time", "key", "subsystem", "msg", "user", "src_ip")} existing.name = kc.get("name")
existing = await session.get(UnifiEvent, uid) existing.ip = kc.get("ip")
if not existing: existing.mac_vendor = kc.get("oui")
session.add(UnifiEvent( existing.note = kc.get("note")
unifi_id=uid, existing.is_blocked = kc.get("blocked", False)
event_time=event_time, existing.first_seen = _ts(kc.get("first_seen"))
event_key=ev.get("key", ""), existing.last_seen = _ts(kc.get("last_seen"))
category=category, existing.scraped_at = scraped_at
message=ev.get("msg", ""), else:
source_mac=ev.get("user") or ev.get("src_mac"), session.add(UnifiKnownClient(
source_ip=ev.get("src_ip"), mac=mac,
details_json=json.dumps(details), hostname=kc.get("hostname"),
)) name=kc.get("name"),
ip=kc.get("ip"),
mac_vendor=kc.get("oui"),
note=kc.get("note"),
is_blocked=kc.get("blocked", False),
first_seen=_ts(kc.get("first_seen")),
last_seen=_ts(kc.get("last_seen")),
scraped_at=scraped_at,
))
# Alarms (upsert) # Speedtest (upsert by run_at)
for al in alarms: if speedtest:
uid = al.get("_id", "") ts_raw = speedtest.get("rundate") or speedtest.get("time")
if not uid: try:
continue if isinstance(ts_raw, (int, float)):
ts_raw = al.get("datetime") or al.get("time") run_at = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
try:
if isinstance(ts_raw, (int, float)):
alarm_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
else:
alarm_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
except Exception:
alarm_time = scraped_at
existing = await session.get(UnifiAlarm, uid)
if existing:
existing.archived = al.get("archived", False)
existing.scraped_at = scraped_at
else: else:
session.add(UnifiAlarm( run_at = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
unifi_id=uid, except Exception:
alarm_time=alarm_time, run_at = scraped_at
alarm_key=al.get("key", ""), existing = await session.get(UnifiSpeedtestResult, run_at)
message=al.get("msg", ""), if not existing:
archived=al.get("archived", False), session.add(UnifiSpeedtestResult(
scraped_at=scraped_at, run_at=run_at,
)) download_mbps=speedtest.get("xput_download"),
upload_mbps=speedtest.get("xput_upload"),
latency_ms=speedtest.get("latency"),
server_name=speedtest.get("server_name"),
))
# Known clients (upsert by MAC) # Networks (upsert by UniFi ID)
for kc in known_clients: for net in networks:
mac = kc.get("mac", "") uid = net.get("_id", "")
if not mac: if not uid:
continue continue
def _ts(val): existing = await session.get(UnifiNetwork, uid)
if not val: if existing:
return None existing.name = net.get("name", "")
try: existing.purpose = net.get("purpose")
if isinstance(val, (int, float)): existing.vlan = net.get("vlan")
return datetime.fromtimestamp(val, tz=timezone.utc) existing.ip_subnet = net.get("ip_subnet")
return datetime.fromisoformat(str(val).replace("Z", "+00:00")) existing.dhcp_enabled = net.get("dhcpd_enabled", False)
except Exception: existing.is_guest = net.get("is_guest", False)
return None existing.scraped_at = scraped_at
existing = await session.get(UnifiKnownClient, mac) else:
if existing: session.add(UnifiNetwork(
existing.hostname = kc.get("hostname") unifi_id=uid,
existing.name = kc.get("name") name=net.get("name", ""),
existing.ip = kc.get("ip") purpose=net.get("purpose"),
existing.mac_vendor = kc.get("oui") vlan=net.get("vlan"),
existing.note = kc.get("note") ip_subnet=net.get("ip_subnet"),
existing.is_blocked = kc.get("blocked", False) dhcp_enabled=net.get("dhcpd_enabled", False),
existing.first_seen = _ts(kc.get("first_seen")) is_guest=net.get("is_guest", False),
existing.last_seen = _ts(kc.get("last_seen")) scraped_at=scraped_at,
existing.scraped_at = scraped_at ))
else:
session.add(UnifiKnownClient(
mac=mac,
hostname=kc.get("hostname"),
name=kc.get("name"),
ip=kc.get("ip"),
mac_vendor=kc.get("oui"),
note=kc.get("note"),
is_blocked=kc.get("blocked", False),
first_seen=_ts(kc.get("first_seen")),
last_seen=_ts(kc.get("last_seen")),
scraped_at=scraped_at,
))
# Speedtest (upsert by run_at) # Port forwards (upsert by UniFi ID)
if speedtest: for pf in port_forwards:
ts_raw = speedtest.get("rundate") or speedtest.get("time") uid = pf.get("_id", "")
try: if not uid:
if isinstance(ts_raw, (int, float)): continue
run_at = datetime.fromtimestamp(ts_raw, tz=timezone.utc) existing = await session.get(UnifiPortForward, uid)
else: if existing:
run_at = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) existing.name = pf.get("name")
except Exception: existing.proto = pf.get("proto", "tcp")
run_at = scraped_at existing.dst_port = pf.get("dst_port", "")
existing = await session.get(UnifiSpeedtestResult, run_at) existing.fwd_ip = pf.get("fwd", "")
if not existing: existing.fwd_port = pf.get("fwd_port", "")
session.add(UnifiSpeedtestResult( existing.src_filter = pf.get("src")
run_at=run_at, existing.enabled = pf.get("enabled", True)
download_mbps=speedtest.get("xput_download"), existing.scraped_at = scraped_at
upload_mbps=speedtest.get("xput_upload"), else:
latency_ms=speedtest.get("latency"), session.add(UnifiPortForward(
server_name=speedtest.get("server_name"), unifi_id=uid,
)) name=pf.get("name"),
proto=pf.get("proto", "tcp"),
dst_port=pf.get("dst_port", ""),
fwd_ip=pf.get("fwd", ""),
fwd_port=pf.get("fwd_port", ""),
src_filter=pf.get("src"),
enabled=pf.get("enabled", True),
scraped_at=scraped_at,
))
# Networks (upsert by UniFi ID) # Firewall rules (upsert by UniFi ID)
for net in networks: for rule in fw_rules:
uid = net.get("_id", "") uid = rule.get("_id", "")
if not uid: if not uid:
continue continue
existing = await session.get(UnifiNetwork, uid) existing = await session.get(UnifiFwRule, uid)
if existing: if existing:
existing.name = net.get("name", "") existing.name = rule.get("name", "")
existing.purpose = net.get("purpose") existing.ruleset = rule.get("ruleset", "")
existing.vlan = net.get("vlan") existing.rule_index = rule.get("rule_index")
existing.ip_subnet = net.get("ip_subnet") existing.action = rule.get("action", "")
existing.dhcp_enabled = net.get("dhcpd_enabled", False) existing.protocol = rule.get("protocol")
existing.is_guest = net.get("is_guest", False) existing.enabled = rule.get("enabled", True)
existing.scraped_at = scraped_at existing.src_address = rule.get("src_address")
else: existing.dst_address = rule.get("dst_address")
session.add(UnifiNetwork( existing.scraped_at = scraped_at
unifi_id=uid, else:
name=net.get("name", ""), session.add(UnifiFwRule(
purpose=net.get("purpose"), unifi_id=uid,
vlan=net.get("vlan"), name=rule.get("name", ""),
ip_subnet=net.get("ip_subnet"), ruleset=rule.get("ruleset", ""),
dhcp_enabled=net.get("dhcpd_enabled", False), rule_index=rule.get("rule_index"),
is_guest=net.get("is_guest", False), action=rule.get("action", ""),
scraped_at=scraped_at, protocol=rule.get("protocol"),
)) enabled=rule.get("enabled", True),
src_address=rule.get("src_address"),
dst_address=rule.get("dst_address"),
scraped_at=scraped_at,
))
# Port forwards (upsert by UniFi ID) await hold_write(app, "UniFi poll", write)
for pf in port_forwards:
uid = pf.get("_id", "")
if not uid:
continue
existing = await session.get(UnifiPortForward, uid)
if existing:
existing.name = pf.get("name")
existing.proto = pf.get("proto", "tcp")
existing.dst_port = pf.get("dst_port", "")
existing.fwd_ip = pf.get("fwd", "")
existing.fwd_port = pf.get("fwd_port", "")
existing.src_filter = pf.get("src")
existing.enabled = pf.get("enabled", True)
existing.scraped_at = scraped_at
else:
session.add(UnifiPortForward(
unifi_id=uid,
name=pf.get("name"),
proto=pf.get("proto", "tcp"),
dst_port=pf.get("dst_port", ""),
fwd_ip=pf.get("fwd", ""),
fwd_port=pf.get("fwd_port", ""),
src_filter=pf.get("src"),
enabled=pf.get("enabled", True),
scraped_at=scraped_at,
))
# Firewall rules (upsert by UniFi ID)
for rule in fw_rules:
uid = rule.get("_id", "")
if not uid:
continue
existing = await session.get(UnifiFwRule, uid)
if existing:
existing.name = rule.get("name", "")
existing.ruleset = rule.get("ruleset", "")
existing.rule_index = rule.get("rule_index")
existing.action = rule.get("action", "")
existing.protocol = rule.get("protocol")
existing.enabled = rule.get("enabled", True)
existing.src_address = rule.get("src_address")
existing.dst_address = rule.get("dst_address")
existing.scraped_at = scraped_at
else:
session.add(UnifiFwRule(
unifi_id=uid,
name=rule.get("name", ""),
ruleset=rule.get("ruleset", ""),
rule_index=rule.get("rule_index"),
action=rule.get("action", ""),
protocol=rule.get("protocol"),
enabled=rule.get("enabled", True),
src_address=rule.get("src_address"),
dst_address=rule.get("dst_address"),
scraped_at=scraped_at,
))
logger.debug( logger.debug(
"UniFi expanded: wlans=%d, events=%d, alarms=%d, known_clients=%d, " "UniFi expanded: wlans=%d, events=%d, alarms=%d, known_clients=%d, "
steward/core/holding.py
+53 -6
View File
@@ -80,13 +80,42 @@ class Holding:
"held (the database may still finish it)", self.name, len(batch), "held (the database may still finish it)", self.name, len(batch),
self.deadline_s) self.deadline_s)
return return
except Exception: except Exception as exc:
logger.exception("%s: storing %d held round(s) failed; they stay held", if unreachable(exc):
logger.warning("%s: the database can't be reached (%s); %d round(s) stay held",
self.name, exc, len(batch))
return
# A round the database refuses would otherwise be retried, and
# refused, ahead of every round after it: store each on its own
# and let go of the ones refused.
logger.exception("%s: storing %d held round(s) failed; storing each alone",
self.name, len(batch)) self.name, len(batch))
return for seq, item in batch:
last = batch[-1][0] try:
while self._held and self._held[0][0] <= last: await asyncio.wait_for(self.store([item]), self.deadline_s)
self._held.popleft() except Exception as one:
if isinstance(one, TimeoutError) or unreachable(one):
return
logger.error("%s: a round collected was refused and is dropped: %s",
self.name, one)
self._drop_through(seq)
continue
self._drop_through(batch[-1][0])
def _drop_through(self, seq: int) -> None:
while self._held and self._held[0][0] <= seq:
self._held.popleft()
def unreachable(exc: BaseException) -> bool:
"""Whether a store failed because the database couldn't be reached, as
opposed to refusing what it was given."""
from sqlalchemy.exc import DBAPIError, InterfaceError, OperationalError
from steward.database import DatabaseUnavailable
if isinstance(exc, (OSError, DatabaseUnavailable, InterfaceError, OperationalError)):
return True
return isinstance(exc, DBAPIError) and bool(exc.connection_invalidated)
def holding(app, name: str, store: Callable[[list], Awaitable[None]], limit: int) -> Holding: def holding(app, name: str, store: Callable[[list], Awaitable[None]], limit: int) -> Holding:
@@ -97,3 +126,21 @@ def holding(app, name: str, store: Callable[[list], Awaitable[None]], limit: int
if name not in holdings: if name not in holdings:
holdings[name] = Holding(name, store, limit) holdings[name] = Holding(name, store, limit)
return holdings[name] return holdings[name]
async def hold_write(app, name: str, write: Callable[[object], Awaitable[None]],
limit: int = 120, wait_s: float = 20.0) -> bool:
"""Hold one collected round as `write(session)` and store it: the held
writes run oldest first, in one transaction. For a collector whose round
is easiest said as the writes it makes. True when it was stored within
`wait_s`."""
held = holding(app, name, store=lambda writes: _run_writes(app, writes), limit=limit)
held.hold(write)
return await held.flush(wait_s)
async def _run_writes(app, writes: list) -> None:
async with app.db_sessionmaker() as session:
async with session.begin():
for write in writes:
await write(session)
steward/monitors/scheduler.py
+98 -37
View File
@@ -5,15 +5,24 @@ enabled monitors and their intervals fresh (a changed interval applies at the
next tick), stamps the due ones with the time they were due to start, and next tick), stamps the due ones with the time they were due to start, and
starts each check on its own. A slow check (a 10 s timeout) holds up nobody starts each check on its own. A slow check (a 10 s timeout) holds up nobody
else, and a monitor still being checked is not started twice. else, and a monitor still being checked is not started twice.
A stall of the database doesn't stop the checks (#5995). The monitor list
is read within MONITORS_READ_S, or the last one read stands in; a stamp the
database doesn't take in that time is kept in memory instead; and each
result is held and stored, oldest first and at the time it was checked, once
the database answers. So a stall shows on the monitors as the checks it saw,
not as a gap.
""" """
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import logging import logging
from datetime import datetime, timezone from datetime import datetime, timezone
from types import SimpleNamespace
from sqlalchemy import select, update from sqlalchemy import select, update
from steward.core.alerts import MetricBatch from steward.core.alerts import MetricBatch
from steward.core.holding import holding
from steward.models.monitors import Monitor, MonitorResult from steward.models.monitors import Monitor, MonitorResult
from .runner import _BLANK, run_monitor from .runner import _BLANK, run_monitor
@@ -25,17 +34,30 @@ DISPATCH_TICK_SECONDS = 5
# A check past this is recorded down rather than left holding its monitor: # A check past this is recorded down rather than left holding its monitor:
# far beyond any probe's own timeout, so it only ever catches a hang. # far beyond any probe's own timeout, so it only ever catches a hang.
CHECK_DEADLINE_SECONDS = 120 CHECK_DEADLINE_SECONDS = 120
# How long the monitor list may take to read, or the stamp to be taken,
# before the last list read and the stamp kept in memory stand in.
MONITORS_READ_S = 10.0
# How long a check waits for its result to be stored before it lets its
# monitor go; the result stays held until the database takes it.
STORE_WAIT_S = 20.0
# Results held while the database doesn't answer: an hour of a minute's
# checks for 60 monitors.
HELD_RESULTS = 3600
def is_due(monitor, now: datetime, global_interval: int) -> bool: def is_due(monitor, now: datetime, global_interval: int,
"""Whether a monitor's interval has elapsed since its last scheduled start. started: datetime | None = None) -> bool:
"""Whether a monitor's interval has elapsed since its last scheduled start:
the stamp stored, or `started`, the one kept in memory when the database
didn't take it.
Half a tick of slack, so a check stamped at t isn't missed at t+interval Half a tick of slack, so a check stamped at t isn't missed at t+interval
by a few milliseconds of tick jitter and pushed back a whole tick.""" by a few milliseconds of tick jitter and pushed back a whole tick."""
if monitor.last_checked_at is None: last = max((t for t in (monitor.last_checked_at, started) if t is not None), default=None)
if last is None:
return True return True
interval = monitor.check_interval_seconds or global_interval interval = monitor.check_interval_seconds or global_interval
elapsed = (now - monitor.last_checked_at).total_seconds() elapsed = (now - last).total_seconds()
return elapsed >= interval - DISPATCH_TICK_SECONDS / 2 return elapsed >= interval - DISPATCH_TICK_SECONDS / 2
@@ -46,6 +68,39 @@ def _state(app) -> tuple[set, set]:
return app._monitor_checks return app._monitor_checks
def _memory(app) -> SimpleNamespace:
"""The monitors as last read, and the starts the database hasn't taken."""
if not hasattr(app, "_monitor_memory"):
app._monitor_memory = SimpleNamespace(monitors=None, started={})
return app._monitor_memory
async def _enabled_monitors(app) -> list:
memory = _memory(app)
async def read() -> list:
async with app.db_sessionmaker() as session:
return list((await session.execute(
select(Monitor).where(Monitor.enabled.is_(True)))).scalars())
try:
memory.monitors = await asyncio.wait_for(read(), MONITORS_READ_S)
except Exception as exc:
if memory.monitors is None:
logger.warning("Monitors not read (%s); nothing to check yet", exc or type(exc).__name__)
return []
logger.warning("Monitors not read within %.0f s (%s); checking the %d last read",
MONITORS_READ_S, exc or type(exc).__name__, len(memory.monitors))
return memory.monitors
async def _stamp(app, ids: list[str], now: datetime) -> None:
async with app.db_sessionmaker() as session:
await session.execute(
update(Monitor).where(Monitor.id.in_(ids)).values(last_checked_at=now))
await session.commit()
async def run_due_monitors(app) -> None: async def run_due_monitors(app) -> None:
"""Start a check for every enabled Monitor that is due and not already """Start a check for every enabled Monitor that is due and not already
being checked. Returns once they are started, not finished. being checked. Returns once they are started, not finished.
@@ -55,19 +110,23 @@ async def run_due_monitors(app) -> None:
global_interval = int(app.config.get("MONITORS_POLL_INTERVAL", 60)) global_interval = int(app.config.get("MONITORS_POLL_INTERVAL", 60))
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
in_flight, tasks = _state(app) in_flight, tasks = _state(app)
memory = _memory(app)
async with app.db_sessionmaker() as session: monitors = await _enabled_monitors(app)
monitors = list((await session.execute( due = [m for m in monitors if m.id not in in_flight
select(Monitor).where(Monitor.enabled.is_(True)) and is_due(m, now, global_interval, memory.started.get(m.id))]
)).scalars()) if not due:
due = [m for m in monitors if m.id not in in_flight and is_due(m, now, global_interval)] return
if not due: # Stamp the scheduled start before checking: the next run is due one
return # interval from now, however long this check takes. Kept in memory too,
# Stamp the scheduled start before checking: the next run is due one # for when the database doesn't take it.
# interval from now, however long this check takes. for m in due:
await session.execute( memory.started[m.id] = now
update(Monitor).where(Monitor.id.in_([m.id for m in due])).values(last_checked_at=now)) try:
await session.commit() await asyncio.wait_for(_stamp(app, [m.id for m in due], now), MONITORS_READ_S)
except Exception as exc:
logger.warning("Monitor starts not stored within %.0f s (%s); kept in memory",
MONITORS_READ_S, exc or type(exc).__name__)
for m in due: for m in due:
in_flight.add(m.id) in_flight.add(m.id)
@@ -77,18 +136,31 @@ async def run_due_monitors(app) -> None:
async def _check_and_record(app, monitor: Monitor) -> None: async def _check_and_record(app, monitor: Monitor) -> None:
"""Run one monitor's check and store its result and metrics.""" """Run one monitor's check, hold its result, and store it."""
in_flight, _ = _state(app) in_flight, _ = _state(app)
try: try:
try: try:
res = await asyncio.wait_for(run_monitor(monitor), CHECK_DEADLINE_SECONDS) res = await asyncio.wait_for(run_monitor(monitor), CHECK_DEADLINE_SECONDS)
except asyncio.TimeoutError: except asyncio.TimeoutError:
res = dict(_BLANK, error_msg=f"Check did not finish within {CHECK_DEADLINE_SECONDS} s") res = dict(_BLANK, error_msg=f"Check did not finish within {CHECK_DEADLINE_SECONDS} s")
check_time = datetime.now(timezone.utc) held = holding(app, "Monitor checks", store=lambda rounds: _store(app, rounds),
async with app.db_sessionmaker() as session: limit=HELD_RESULTS)
async with session.begin(): held.hold((monitor.id, monitor.type, monitor.name, datetime.now(timezone.utc), res))
await held.flush(STORE_WAIT_S)
except Exception:
logger.exception("Monitor %r check errored", monitor.name)
finally:
in_flight.discard(monitor.id)
async def _store(app, checks: list) -> None:
"""Write held check results, oldest first, each at the time it was checked."""
async with app.db_sessionmaker() as session:
async with session.begin():
batch = MetricBatch()
for monitor_id, mtype, name, check_time, res in checks:
session.add(MonitorResult( session.add(MonitorResult(
monitor_id=monitor.id, monitor_id=monitor_id,
checked_at=check_time, checked_at=check_time,
is_up=res["is_up"], is_up=res["is_up"],
response_ms=res["response_ms"], response_ms=res["response_ms"],
@@ -99,20 +171,9 @@ async def _check_and_record(app, monitor: Monitor) -> None:
error_msg=res["error_msg"], error_msg=res["error_msg"],
)) ))
# Alert pipeline — keyed by monitor type (source) + name. # Alert pipeline — keyed by monitor type (source) + name.
batch = MetricBatch() batch.add(source_module=mtype, resource_name=name, metric_name="is_up",
batch.add( value=1.0 if res["is_up"] else 0.0, recorded_at=check_time)
source_module=monitor.type,
resource_name=monitor.name, metric_name="is_up",
value=1.0 if res["is_up"] else 0.0,
)
if res["response_ms"] is not None: if res["response_ms"] is not None:
batch.add( batch.add(source_module=mtype, resource_name=name, metric_name="response_ms",
source_module=monitor.type, value=res["response_ms"], recorded_at=check_time)
resource_name=monitor.name, metric_name="response_ms", await batch.record(session)
value=res["response_ms"],
)
await batch.record(session)
except Exception:
logger.exception("Monitor %r check errored", monitor.name)
finally:
in_flight.discard(monitor.id)
tests/core/test_holding.py
+23 -9
View File
@@ -34,15 +34,12 @@ def test_past_its_limit_the_oldest_round_goes_and_is_counted():
assert len(held) == 2 and held.dropped == 1 and [i for _, i in held._held] == ["b", "c"] assert len(held) == 2 and held.dropped == 1 and [i for _, i in held._held] == ["b", "c"]
def test_a_store_that_fails_or_overruns_its_deadline_keeps_the_rounds_held(): def test_a_store_that_cant_reach_the_database_or_overruns_its_deadline_keeps_the_rounds_held():
async def main(): async def main():
calls = [] async def unreachable(rounds):
raise ConnectionRefusedError("database down")
async def failing(rounds): held = Holding("t", unreachable, limit=10)
calls.append(list(rounds))
raise RuntimeError("database refused")
held = Holding("t", failing, limit=10)
held.hold("a") held.hold("a")
assert await held.flush(1.0) is False and len(held) == 1 assert await held.flush(1.0) is False and len(held) == 1
@@ -52,9 +49,26 @@ def test_a_store_that_fails_or_overruns_its_deadline_keeps_the_rounds_held():
stuck = Holding("t", never, limit=10, deadline_s=0.05) stuck = Holding("t", never, limit=10, deadline_s=0.05)
stuck.hold("a") stuck.hold("a")
assert await stuck.flush(1.0) is False and len(stuck) == 1 # given up, not lost assert await stuck.flush(1.0) is False and len(stuck) == 1 # given up, not lost
return calls
assert asyncio.run(main()) == [["a"]] asyncio.run(main())
def test_a_round_the_database_refuses_is_dropped_and_holds_up_nobody_after_it():
async def main():
stored = []
async def store(rounds):
if "bad" in rounds:
raise ValueError("value out of range")
stored.extend(rounds)
held = Holding("t", store, limit=10)
for r in ("a", "bad", "c"):
held.hold(r)
assert await held.flush(1.0) is True
return stored
assert asyncio.run(main()) == ["a", "c"]
def test_an_app_has_one_holding_per_collector(): def test_an_app_has_one_holding_per_collector():
tests/core/test_monitor_schedule.py
+38
View File
@@ -152,3 +152,41 @@ async def test_nothing_due_stamps_nothing(monkeypatch):
app = _app(store) app = _app(store)
await sched.run_due_monitors(app) await sched.run_due_monitors(app)
assert store.stamped == [] and store.added == [] assert store.stamped == [] and store.added == []
class _HungSession(_Session):
"""The storage under Postgres has stalled: nothing answers until `gate`."""
def __init__(self, store, gate):
super().__init__(store)
self.gate = gate
async def __aenter__(self):
await self.gate.wait()
return self
async def test_checks_go_on_while_the_database_hangs_and_are_stored_at_their_own_time(monkeypatch):
async def fake_run(m):
return dict(sched._BLANK, is_up=False, error_msg="502 Bad Gateway")
monkeypatch.setattr(sched, "run_monitor", fake_run)
monkeypatch.setattr(sched, "MetricBatch", _Batch)
monkeypatch.setattr(sched, "MONITORS_READ_S", 0.05)
monkeypatch.setattr(sched, "STORE_WAIT_S", 0.05)
store, gate = _Store([_m("app")]), asyncio.Event()
app = NS(config={"MONITORS_POLL_INTERVAL": 60}, db_sessionmaker=lambda: _HungSession(store, gate))
sched._memory(app).monitors = list(store.monitors) # as last read, before the stall
await sched.run_due_monitors(app) # the list and the stamp time out
await _settle(app) # the result is held, the monitor freed
assert store.added == [] and sched._state(app)[0] == set()
await sched.run_due_monitors(app)
await _settle(app)
assert store.added == [] # its start was kept in memory: not due again
checked_by = datetime.now(timezone.utc)
gate.set() # the database answers
await app._holdings["Monitor checks"].flush(1.0)
[res] = store.added
assert res.is_up is False and res.checked_at < checked_by # stored at the time it was checked