bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: monitors, UniFi and Traefik keep collecting through a storage stall (#5995)
CI / lint (push) Successful in 3s
Details
CI / integration (push) Successful in 1m38s
Details
CI / unit (push) Successful in 1m39s
Details
CI / publish (push) Successful in 1m11s
Details

Browse Source 
The same Holding as the SNMP poll, for the rest of Steward's own
collectors:

- Monitors: the list is read within 10 s or the last one read stands in;
  a start the database doesn't take is kept in memory, so a monitor isn't
  checked twice; each result is held and stored at the time it was
  checked. A stall shows on the monitors as the checks it saw, not a gap.
- UniFi and Traefik: each poll's writes are held (hold_write) and run
  oldest first in one transaction; every reading is stamped with its scrape.
- A held round the database refuses (as opposed to can't be reached) is
  stored alone and let go, so it can't hold up every round after it.

The Prometheus scrape is left as it is: its one target, Fabled Archives,
keeps its own database on the same NFS.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-17 09:41:14 -04:00
co-authored by Claude Opus 5.5
parent 41fc8073db
commit 2df8431de9
6 changed files with 557 additions and 388 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/traefik/scheduler.py
+73 -70
View File
@@ -75,6 +75,7 @@ async def _do_scrape(app: "Quart") -> None:
fetch_router_domains,
)
from steward.core.alerts import MetricBatch
from steward.core.holding import hold_write
metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"]
@@ -106,81 +107,83 @@ async def _do_scrape(app: "Quart") -> None:
scraped_at = datetime.now(timezone.utc)
async with app.db_sessionmaker() as session:
async with session.begin():
batch = MetricBatch()
# ── Per-service metrics ───────────────────────────────────────────
for router_name, metrics in router_metrics.items():
row = TraefikMetric(
router_name=router_name,
scraped_at=scraped_at,
request_rate=metrics["request_rate"],
error_rate_4xx_pct=metrics["error_rate_4xx_pct"],
error_rate_5xx_pct=metrics["error_rate_5xx_pct"],
latency_p50_ms=metrics["latency_p50_ms"],
latency_p95_ms=metrics["latency_p95_ms"],
latency_p99_ms=metrics["latency_p99_ms"],
response_bytes_rate=metrics["response_bytes_rate"],
)
session.add(row)
async def write(session) -> None:
batch = MetricBatch()
# ── Per-service metrics ───────────────────────────────────────────
for router_name, metrics in router_metrics.items():
row = TraefikMetric(
router_name=router_name,
scraped_at=scraped_at,
request_rate=metrics["request_rate"],
error_rate_4xx_pct=metrics["error_rate_4xx_pct"],
error_rate_5xx_pct=metrics["error_rate_5xx_pct"],
latency_p50_ms=metrics["latency_p50_ms"],
latency_p95_ms=metrics["latency_p95_ms"],
latency_p99_ms=metrics["latency_p99_ms"],
response_bytes_rate=metrics["response_bytes_rate"],
)
session.add(row)
for metric_name, value in [
("request_rate", metrics["request_rate"]),
("error_rate_4xx_pct", metrics["error_rate_4xx_pct"]),
("error_rate_5xx_pct", metrics["error_rate_5xx_pct"]),
("latency_p50_ms", metrics["latency_p50_ms"]),
("latency_p95_ms", metrics["latency_p95_ms"]),
("latency_p99_ms", metrics["latency_p99_ms"]),
("response_bytes_rate", metrics["response_bytes_rate"]),
]:
if value is None: # no responses since the last scrape: no latency
continue
# Stamped with the scrape, so a router's latest figures
# (routes.latest_per_router) all carry the same moment.
batch.add(
source_module="traefik",
resource_name=router_name,
metric_name=metric_name,
value=value,
recorded_at=scraped_at,
)
# ── 4xx and 5xx responses by code (#5711) ─────────────────────────
for router_name, codes in status_codes.items():
for code, count in codes.items():
session.add(TraefikStatusCode(router_name=router_name, scraped_at=scraped_at,
code=code, count=count))
# ── Global stats ──────────────────────────────────────────────────
session.add(TraefikGlobalStat(scraped_at=scraped_at, **global_stats))
# ── TLS certs (upsert by serial) ──────────────────────────────────
for cert in cert_list:
existing = await session.get(TraefikCert, cert["serial"])
if existing:
existing.cn = cert["cn"]
existing.sans = cert["sans"]
existing.not_after = cert["not_after"]
existing.scraped_at = scraped_at
else:
session.add(TraefikCert(
serial=cert["serial"],
cn=cert["cn"],
sans=cert["sans"],
not_after=cert["not_after"],
scraped_at=scraped_at,
))
# Emit cert_expiry_days for alert rules
days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0
for metric_name, value in [
("request_rate", metrics["request_rate"]),
("error_rate_4xx_pct", metrics["error_rate_4xx_pct"]),
("error_rate_5xx_pct", metrics["error_rate_5xx_pct"]),
("latency_p50_ms", metrics["latency_p50_ms"]),
("latency_p95_ms", metrics["latency_p95_ms"]),
("latency_p99_ms", metrics["latency_p99_ms"]),
("response_bytes_rate", metrics["response_bytes_rate"]),
]:
if value is None: # no responses since the last scrape: no latency
continue
# Stamped with the scrape, so a router's latest figures
# (routes.latest_per_router) all carry the same moment.
batch.add(
source_module="traefik",
resource_name=cert["cn"] or cert["serial"],
metric_name="cert_expiry_days",
value=max(0.0, days_remaining),
resource_name=router_name,
metric_name=metric_name,
value=value,
recorded_at=scraped_at,
)
await batch.record(session)
# ── 4xx and 5xx responses by code (#5711) ─────────────────────────
for router_name, codes in status_codes.items():
for code, count in codes.items():
session.add(TraefikStatusCode(router_name=router_name, scraped_at=scraped_at,
code=code, count=count))
# ── Global stats ──────────────────────────────────────────────────
session.add(TraefikGlobalStat(scraped_at=scraped_at, **global_stats))
# ── TLS certs (upsert by serial) ──────────────────────────────────
for cert in cert_list:
existing = await session.get(TraefikCert, cert["serial"])
if existing:
existing.cn = cert["cn"]
existing.sans = cert["sans"]
existing.not_after = cert["not_after"]
existing.scraped_at = scraped_at
else:
session.add(TraefikCert(
serial=cert["serial"],
cn=cert["cn"],
sans=cert["sans"],
not_after=cert["not_after"],
scraped_at=scraped_at,
))
# Emit cert_expiry_days for alert rules
days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0
batch.add(
source_module="traefik",
resource_name=cert["cn"] or cert["serial"],
metric_name="cert_expiry_days",
value=max(0.0, days_remaining),
recorded_at=scraped_at,
)
await batch.record(session)
await hold_write(app, "Traefik scrape", write)
logger.debug(
"Traefik scrape: %d router(s), %d cert(s)",
plugins/unifi/scheduler.py
+272 -266
View File
@@ -5,6 +5,7 @@ import logging
from datetime import datetime, timezone
from typing import TYPE_CHECKING
from steward.core.holding import hold_write
from steward.core.scheduler import ScheduledTask
if TYPE_CHECKING:
@@ -130,68 +131,72 @@ async def _do_poll(app: "Quart") -> None:
top_clients_json=json.dumps(top_clients_data),
)
async with app.db_sessionmaker() as session:
async with session.begin():
session.add(wan_stat)
session.add(client_snapshot)
async def write(session) -> None:
session.add(wan_stat)
session.add(client_snapshot)
# ── Devices (upsert by MAC) ───────────────────────────────────────
for d in devices:
mac = d.get("mac", "")
if not mac:
continue
last_seen_ts = d.get("last_seen")
last_seen_dt = (
datetime.fromtimestamp(last_seen_ts, tz=timezone.utc)
if last_seen_ts else None
)
existing = await session.get(UnifiDevice, mac)
if existing:
existing.name = d.get("name")
existing.model = d.get("model")
existing.device_type = d.get("type")
existing.ip = d.get("ip")
existing.state = d.get("state", 0)
existing.uptime_seconds = d.get("uptime")
existing.last_seen = last_seen_dt
existing.version = d.get("version")
existing.scraped_at = scraped_at
else:
session.add(UnifiDevice(
mac=mac,
name=d.get("name"),
model=d.get("model"),
device_type=d.get("type"),
ip=d.get("ip"),
state=d.get("state", 0),
uptime_seconds=d.get("uptime"),
last_seen=last_seen_dt,
version=d.get("version"),
scraped_at=scraped_at,
))
# ── Devices (upsert by MAC) ───────────────────────────────────────
for d in devices:
mac = d.get("mac", "")
if not mac:
continue
last_seen_ts = d.get("last_seen")
last_seen_dt = (
datetime.fromtimestamp(last_seen_ts, tz=timezone.utc)
if last_seen_ts else None
)
existing = await session.get(UnifiDevice, mac)
if existing:
existing.name = d.get("name")
existing.model = d.get("model")
existing.device_type = d.get("type")
existing.ip = d.get("ip")
existing.state = d.get("state", 0)
existing.uptime_seconds = d.get("uptime")
existing.last_seen = last_seen_dt
existing.version = d.get("version")
existing.scraped_at = scraped_at
else:
session.add(UnifiDevice(
mac=mac,
name=d.get("name"),
model=d.get("model"),
device_type=d.get("type"),
ip=d.get("ip"),
state=d.get("state", 0),
uptime_seconds=d.get("uptime"),
last_seen=last_seen_dt,
version=d.get("version"),
scraped_at=scraped_at,
))
# ── Alert metrics ─────────────────────────────────────────────────
batch = MetricBatch()
if wan_stat.latency_ms is not None:
batch.add(
source_module="unifi",
resource_name="wan",
metric_name="latency_ms",
value=wan_stat.latency_ms,
)
# ── Alert metrics ─────────────────────────────────────────────────
batch = MetricBatch()
if wan_stat.latency_ms is not None:
batch.add(
source_module="unifi",
resource_name="wan",
metric_name="is_up",
value=1.0 if wan_stat.is_up else 0.0,
metric_name="latency_ms",
value=wan_stat.latency_ms,
recorded_at=scraped_at,
)
batch.add(
source_module="unifi",
resource_name="clients",
metric_name="total_clients",
value=float(len(clients)),
)
await batch.record(session)
batch.add(
source_module="unifi",
resource_name="wan",
metric_name="is_up",
value=1.0 if wan_stat.is_up else 0.0,
recorded_at=scraped_at,
)
batch.add(
source_module="unifi",
resource_name="clients",
metric_name="total_clients",
value=float(len(clients)),
recorded_at=scraped_at,
)
await batch.record(session)
await hold_write(app, "UniFi poll", write)
logger.debug(
"UniFi poll: wan=%s, clients=%d, devices=%d",
@@ -310,227 +315,228 @@ async def _do_expanded(app: "Quart", scraped_at: datetime) -> None:
top_dpi_clients = sorted(client_dpi.values(), key=lambda x: x["bytes"], reverse=True)[:10]
# ── Write to DB ───────────────────────────────────────────────────────────
async with app.db_sessionmaker() as session:
async with session.begin():
# WLAN stats
for row in wlan_rows:
session.add(row)
async def write(session) -> None:
# WLAN stats
for row in wlan_rows:
session.add(row)
# DPI snapshot (only if we got data)
if cat_summary or top_dpi_clients:
session.add(UnifiDpiSnapshot(
scraped_at=scraped_at,
by_category_json=json.dumps(cat_summary),
by_client_json=json.dumps(top_dpi_clients),
# DPI snapshot (only if we got data)
if cat_summary or top_dpi_clients:
session.add(UnifiDpiSnapshot(
scraped_at=scraped_at,
by_category_json=json.dumps(cat_summary),
by_client_json=json.dumps(top_dpi_clients),
))
# Events (upsert by _id)
_category_map = {
"wlan": "wlan", "wireless": "wlan",
"wired": "wired",
"ugw": "wan", "wan": "wan",
"ids": "ids",
"system": "system",
"ap": "ap",
}
for ev in events:
uid = ev.get("_id", "")
if not uid:
continue
ts_raw = ev.get("datetime") or ev.get("time")
try:
if isinstance(ts_raw, (int, float)):
event_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
else:
event_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
except Exception:
event_time = scraped_at
subsystem = ev.get("subsystem", "system")
category = _category_map.get(subsystem, subsystem[:16])
details = {k: v for k, v in ev.items()
if k not in ("_id", "datetime", "time", "key", "subsystem", "msg", "user", "src_ip")}
existing = await session.get(UnifiEvent, uid)
if not existing:
session.add(UnifiEvent(
unifi_id=uid,
event_time=event_time,
event_key=ev.get("key", ""),
category=category,
message=ev.get("msg", ""),
source_mac=ev.get("user") or ev.get("src_mac"),
source_ip=ev.get("src_ip"),
details_json=json.dumps(details),
))
# Events (upsert by _id)
_category_map = {
"wlan": "wlan", "wireless": "wlan",
"wired": "wired",
"ugw": "wan", "wan": "wan",
"ids": "ids",
"system": "system",
"ap": "ap",
}
for ev in events:
uid = ev.get("_id", "")
if not uid:
continue
ts_raw = ev.get("datetime") or ev.get("time")
# Alarms (upsert)
for al in alarms:
uid = al.get("_id", "")
if not uid:
continue
ts_raw = al.get("datetime") or al.get("time")
try:
if isinstance(ts_raw, (int, float)):
alarm_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
else:
alarm_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
except Exception:
alarm_time = scraped_at
existing = await session.get(UnifiAlarm, uid)
if existing:
existing.archived = al.get("archived", False)
existing.scraped_at = scraped_at
else:
session.add(UnifiAlarm(
unifi_id=uid,
alarm_time=alarm_time,
alarm_key=al.get("key", ""),
message=al.get("msg", ""),
archived=al.get("archived", False),
scraped_at=scraped_at,
))
# Known clients (upsert by MAC)
for kc in known_clients:
mac = kc.get("mac", "")
if not mac:
continue
def _ts(val):
if not val:
return None
try:
if isinstance(ts_raw, (int, float)):
event_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
else:
event_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
if isinstance(val, (int, float)):
return datetime.fromtimestamp(val, tz=timezone.utc)
return datetime.fromisoformat(str(val).replace("Z", "+00:00"))
except Exception:
event_time = scraped_at
subsystem = ev.get("subsystem", "system")
category = _category_map.get(subsystem, subsystem[:16])
details = {k: v for k, v in ev.items()
if k not in ("_id", "datetime", "time", "key", "subsystem", "msg", "user", "src_ip")}
existing = await session.get(UnifiEvent, uid)
if not existing:
session.add(UnifiEvent(
unifi_id=uid,
event_time=event_time,
event_key=ev.get("key", ""),
category=category,
message=ev.get("msg", ""),
source_mac=ev.get("user") or ev.get("src_mac"),
source_ip=ev.get("src_ip"),
details_json=json.dumps(details),
))
return None
existing = await session.get(UnifiKnownClient, mac)
if existing:
existing.hostname = kc.get("hostname")
existing.name = kc.get("name")
existing.ip = kc.get("ip")
existing.mac_vendor = kc.get("oui")
existing.note = kc.get("note")
existing.is_blocked = kc.get("blocked", False)
existing.first_seen = _ts(kc.get("first_seen"))
existing.last_seen = _ts(kc.get("last_seen"))
existing.scraped_at = scraped_at
else:
session.add(UnifiKnownClient(
mac=mac,
hostname=kc.get("hostname"),
name=kc.get("name"),
ip=kc.get("ip"),
mac_vendor=kc.get("oui"),
note=kc.get("note"),
is_blocked=kc.get("blocked", False),
first_seen=_ts(kc.get("first_seen")),
last_seen=_ts(kc.get("last_seen")),
scraped_at=scraped_at,
))
# Alarms (upsert)
for al in alarms:
uid = al.get("_id", "")
if not uid:
continue
ts_raw = al.get("datetime") or al.get("time")
try:
if isinstance(ts_raw, (int, float)):
alarm_time = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
else:
alarm_time = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
except Exception:
alarm_time = scraped_at
existing = await session.get(UnifiAlarm, uid)
if existing:
existing.archived = al.get("archived", False)
existing.scraped_at = scraped_at
# Speedtest (upsert by run_at)
if speedtest:
ts_raw = speedtest.get("rundate") or speedtest.get("time")
try:
if isinstance(ts_raw, (int, float)):
run_at = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
else:
session.add(UnifiAlarm(
unifi_id=uid,
alarm_time=alarm_time,
alarm_key=al.get("key", ""),
message=al.get("msg", ""),
archived=al.get("archived", False),
scraped_at=scraped_at,
))
run_at = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
except Exception:
run_at = scraped_at
existing = await session.get(UnifiSpeedtestResult, run_at)
if not existing:
session.add(UnifiSpeedtestResult(
run_at=run_at,
download_mbps=speedtest.get("xput_download"),
upload_mbps=speedtest.get("xput_upload"),
latency_ms=speedtest.get("latency"),
server_name=speedtest.get("server_name"),
))
# Known clients (upsert by MAC)
for kc in known_clients:
mac = kc.get("mac", "")
if not mac:
continue
def _ts(val):
if not val:
return None
try:
if isinstance(val, (int, float)):
return datetime.fromtimestamp(val, tz=timezone.utc)
return datetime.fromisoformat(str(val).replace("Z", "+00:00"))
except Exception:
return None
existing = await session.get(UnifiKnownClient, mac)
if existing:
existing.hostname = kc.get("hostname")
existing.name = kc.get("name")
existing.ip = kc.get("ip")
existing.mac_vendor = kc.get("oui")
existing.note = kc.get("note")
existing.is_blocked = kc.get("blocked", False)
existing.first_seen = _ts(kc.get("first_seen"))
existing.last_seen = _ts(kc.get("last_seen"))
existing.scraped_at = scraped_at
else:
session.add(UnifiKnownClient(
mac=mac,
hostname=kc.get("hostname"),
name=kc.get("name"),
ip=kc.get("ip"),
mac_vendor=kc.get("oui"),
note=kc.get("note"),
is_blocked=kc.get("blocked", False),
first_seen=_ts(kc.get("first_seen")),
last_seen=_ts(kc.get("last_seen")),
scraped_at=scraped_at,
))
# Networks (upsert by UniFi ID)
for net in networks:
uid = net.get("_id", "")
if not uid:
continue
existing = await session.get(UnifiNetwork, uid)
if existing:
existing.name = net.get("name", "")
existing.purpose = net.get("purpose")
existing.vlan = net.get("vlan")
existing.ip_subnet = net.get("ip_subnet")
existing.dhcp_enabled = net.get("dhcpd_enabled", False)
existing.is_guest = net.get("is_guest", False)
existing.scraped_at = scraped_at
else:
session.add(UnifiNetwork(
unifi_id=uid,
name=net.get("name", ""),
purpose=net.get("purpose"),
vlan=net.get("vlan"),
ip_subnet=net.get("ip_subnet"),
dhcp_enabled=net.get("dhcpd_enabled", False),
is_guest=net.get("is_guest", False),
scraped_at=scraped_at,
))
# Speedtest (upsert by run_at)
if speedtest:
ts_raw = speedtest.get("rundate") or speedtest.get("time")
try:
if isinstance(ts_raw, (int, float)):
run_at = datetime.fromtimestamp(ts_raw, tz=timezone.utc)
else:
run_at = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
except Exception:
run_at = scraped_at
existing = await session.get(UnifiSpeedtestResult, run_at)
if not existing:
session.add(UnifiSpeedtestResult(
run_at=run_at,
download_mbps=speedtest.get("xput_download"),
upload_mbps=speedtest.get("xput_upload"),
latency_ms=speedtest.get("latency"),
server_name=speedtest.get("server_name"),
))
# Port forwards (upsert by UniFi ID)
for pf in port_forwards:
uid = pf.get("_id", "")
if not uid:
continue
existing = await session.get(UnifiPortForward, uid)
if existing:
existing.name = pf.get("name")
existing.proto = pf.get("proto", "tcp")
existing.dst_port = pf.get("dst_port", "")
existing.fwd_ip = pf.get("fwd", "")
existing.fwd_port = pf.get("fwd_port", "")
existing.src_filter = pf.get("src")
existing.enabled = pf.get("enabled", True)
existing.scraped_at = scraped_at
else:
session.add(UnifiPortForward(
unifi_id=uid,
name=pf.get("name"),
proto=pf.get("proto", "tcp"),
dst_port=pf.get("dst_port", ""),
fwd_ip=pf.get("fwd", ""),
fwd_port=pf.get("fwd_port", ""),
src_filter=pf.get("src"),
enabled=pf.get("enabled", True),
scraped_at=scraped_at,
))
# Networks (upsert by UniFi ID)
for net in networks:
uid = net.get("_id", "")
if not uid:
continue
existing = await session.get(UnifiNetwork, uid)
if existing:
existing.name = net.get("name", "")
existing.purpose = net.get("purpose")
existing.vlan = net.get("vlan")
existing.ip_subnet = net.get("ip_subnet")
existing.dhcp_enabled = net.get("dhcpd_enabled", False)
existing.is_guest = net.get("is_guest", False)
existing.scraped_at = scraped_at
else:
session.add(UnifiNetwork(
unifi_id=uid,
name=net.get("name", ""),
purpose=net.get("purpose"),
vlan=net.get("vlan"),
ip_subnet=net.get("ip_subnet"),
dhcp_enabled=net.get("dhcpd_enabled", False),
is_guest=net.get("is_guest", False),
scraped_at=scraped_at,
))
# Firewall rules (upsert by UniFi ID)
for rule in fw_rules:
uid = rule.get("_id", "")
if not uid:
continue
existing = await session.get(UnifiFwRule, uid)
if existing:
existing.name = rule.get("name", "")
existing.ruleset = rule.get("ruleset", "")
existing.rule_index = rule.get("rule_index")
existing.action = rule.get("action", "")
existing.protocol = rule.get("protocol")
existing.enabled = rule.get("enabled", True)
existing.src_address = rule.get("src_address")
existing.dst_address = rule.get("dst_address")
existing.scraped_at = scraped_at
else:
session.add(UnifiFwRule(
unifi_id=uid,
name=rule.get("name", ""),
ruleset=rule.get("ruleset", ""),
rule_index=rule.get("rule_index"),
action=rule.get("action", ""),
protocol=rule.get("protocol"),
enabled=rule.get("enabled", True),
src_address=rule.get("src_address"),
dst_address=rule.get("dst_address"),
scraped_at=scraped_at,
))
# Port forwards (upsert by UniFi ID)
for pf in port_forwards:
uid = pf.get("_id", "")
if not uid:
continue
existing = await session.get(UnifiPortForward, uid)
if existing:
existing.name = pf.get("name")
existing.proto = pf.get("proto", "tcp")
existing.dst_port = pf.get("dst_port", "")
existing.fwd_ip = pf.get("fwd", "")
existing.fwd_port = pf.get("fwd_port", "")
existing.src_filter = pf.get("src")
existing.enabled = pf.get("enabled", True)
existing.scraped_at = scraped_at
else:
session.add(UnifiPortForward(
unifi_id=uid,
name=pf.get("name"),
proto=pf.get("proto", "tcp"),
dst_port=pf.get("dst_port", ""),
fwd_ip=pf.get("fwd", ""),
fwd_port=pf.get("fwd_port", ""),
src_filter=pf.get("src"),
enabled=pf.get("enabled", True),
scraped_at=scraped_at,
))
# Firewall rules (upsert by UniFi ID)
for rule in fw_rules:
uid = rule.get("_id", "")
if not uid:
continue
existing = await session.get(UnifiFwRule, uid)
if existing:
existing.name = rule.get("name", "")
existing.ruleset = rule.get("ruleset", "")
existing.rule_index = rule.get("rule_index")
existing.action = rule.get("action", "")
existing.protocol = rule.get("protocol")
existing.enabled = rule.get("enabled", True)
existing.src_address = rule.get("src_address")
existing.dst_address = rule.get("dst_address")
existing.scraped_at = scraped_at
else:
session.add(UnifiFwRule(
unifi_id=uid,
name=rule.get("name", ""),
ruleset=rule.get("ruleset", ""),
rule_index=rule.get("rule_index"),
action=rule.get("action", ""),
protocol=rule.get("protocol"),
enabled=rule.get("enabled", True),
src_address=rule.get("src_address"),
dst_address=rule.get("dst_address"),
scraped_at=scraped_at,
))
await hold_write(app, "UniFi poll", write)
logger.debug(
"UniFi expanded: wlans=%d, events=%d, alarms=%d, known_clients=%d, "
steward/core/holding.py
+53 -6
View File
@@ -80,13 +80,42 @@ class Holding:
"held (the database may still finish it)", self.name, len(batch),
self.deadline_s)
return
except Exception:
logger.exception("%s: storing %d held round(s) failed; they stay held",
except Exception as exc:
if unreachable(exc):
logger.warning("%s: the database can't be reached (%s); %d round(s) stay held",
self.name, exc, len(batch))
return
# A round the database refuses would otherwise be retried, and
# refused, ahead of every round after it: store each on its own
# and let go of the ones refused.
logger.exception("%s: storing %d held round(s) failed; storing each alone",
self.name, len(batch))
return
last = batch[-1][0]
while self._held and self._held[0][0] <= last:
self._held.popleft()
for seq, item in batch:
try:
await asyncio.wait_for(self.store([item]), self.deadline_s)
except Exception as one:
if isinstance(one, TimeoutError) or unreachable(one):
return
logger.error("%s: a round collected was refused and is dropped: %s",
self.name, one)
self._drop_through(seq)
continue
self._drop_through(batch[-1][0])
def _drop_through(self, seq: int) -> None:
while self._held and self._held[0][0] <= seq:
self._held.popleft()
def unreachable(exc: BaseException) -> bool:
"""Whether a store failed because the database couldn't be reached, as
opposed to refusing what it was given."""
from sqlalchemy.exc import DBAPIError, InterfaceError, OperationalError
from steward.database import DatabaseUnavailable
if isinstance(exc, (OSError, DatabaseUnavailable, InterfaceError, OperationalError)):
return True
return isinstance(exc, DBAPIError) and bool(exc.connection_invalidated)
def holding(app, name: str, store: Callable[[list], Awaitable[None]], limit: int) -> Holding:
@@ -97,3 +126,21 @@ def holding(app, name: str, store: Callable[[list], Awaitable[None]], limit: int
if name not in holdings:
holdings[name] = Holding(name, store, limit)
return holdings[name]
async def hold_write(app, name: str, write: Callable[[object], Awaitable[None]],
limit: int = 120, wait_s: float = 20.0) -> bool:
"""Hold one collected round as `write(session)` and store it: the held
writes run oldest first, in one transaction. For a collector whose round
is easiest said as the writes it makes. True when it was stored within
`wait_s`."""
held = holding(app, name, store=lambda writes: _run_writes(app, writes), limit=limit)
held.hold(write)
return await held.flush(wait_s)
async def _run_writes(app, writes: list) -> None:
async with app.db_sessionmaker() as session:
async with session.begin():
for write in writes:
await write(session)
steward/monitors/scheduler.py
+98 -37
View File
@@ -5,15 +5,24 @@ enabled monitors and their intervals fresh (a changed interval applies at the
next tick), stamps the due ones with the time they were due to start, and
starts each check on its own. A slow check (a 10 s timeout) holds up nobody
else, and a monitor still being checked is not started twice.
A stall of the database doesn't stop the checks (#5995). The monitor list
is read within MONITORS_READ_S, or the last one read stands in; a stamp the
database doesn't take in that time is kept in memory instead; and each
result is held and stored, oldest first and at the time it was checked, once
the database answers. So a stall shows on the monitors as the checks it saw,
not as a gap.
"""
from __future__ import annotations
import asyncio
import logging
from datetime import datetime, timezone
from types import SimpleNamespace
from sqlalchemy import select, update
from steward.core.alerts import MetricBatch
from steward.core.holding import holding
from steward.models.monitors import Monitor, MonitorResult
from .runner import _BLANK, run_monitor
@@ -25,17 +34,30 @@ DISPATCH_TICK_SECONDS = 5
# A check past this is recorded down rather than left holding its monitor:
# far beyond any probe's own timeout, so it only ever catches a hang.
CHECK_DEADLINE_SECONDS = 120
# How long the monitor list may take to read, or the stamp to be taken,
# before the last list read and the stamp kept in memory stand in.
MONITORS_READ_S = 10.0
# How long a check waits for its result to be stored before it lets its
# monitor go; the result stays held until the database takes it.
STORE_WAIT_S = 20.0
# Results held while the database doesn't answer: an hour of a minute's
# checks for 60 monitors.
HELD_RESULTS = 3600
def is_due(monitor, now: datetime, global_interval: int) -> bool:
"""Whether a monitor's interval has elapsed since its last scheduled start.
def is_due(monitor, now: datetime, global_interval: int,
started: datetime | None = None) -> bool:
"""Whether a monitor's interval has elapsed since its last scheduled start:
the stamp stored, or `started`, the one kept in memory when the database
didn't take it.
Half a tick of slack, so a check stamped at t isn't missed at t+interval
by a few milliseconds of tick jitter and pushed back a whole tick."""
if monitor.last_checked_at is None:
last = max((t for t in (monitor.last_checked_at, started) if t is not None), default=None)
if last is None:
return True
interval = monitor.check_interval_seconds or global_interval
elapsed = (now - monitor.last_checked_at).total_seconds()
elapsed = (now - last).total_seconds()
return elapsed >= interval - DISPATCH_TICK_SECONDS / 2
@@ -46,6 +68,39 @@ def _state(app) -> tuple[set, set]:
return app._monitor_checks
def _memory(app) -> SimpleNamespace:
"""The monitors as last read, and the starts the database hasn't taken."""
if not hasattr(app, "_monitor_memory"):
app._monitor_memory = SimpleNamespace(monitors=None, started={})
return app._monitor_memory
async def _enabled_monitors(app) -> list:
memory = _memory(app)
async def read() -> list:
async with app.db_sessionmaker() as session:
return list((await session.execute(
select(Monitor).where(Monitor.enabled.is_(True)))).scalars())
try:
memory.monitors = await asyncio.wait_for(read(), MONITORS_READ_S)
except Exception as exc:
if memory.monitors is None:
logger.warning("Monitors not read (%s); nothing to check yet", exc or type(exc).__name__)
return []
logger.warning("Monitors not read within %.0f s (%s); checking the %d last read",
MONITORS_READ_S, exc or type(exc).__name__, len(memory.monitors))
return memory.monitors
async def _stamp(app, ids: list[str], now: datetime) -> None:
async with app.db_sessionmaker() as session:
await session.execute(
update(Monitor).where(Monitor.id.in_(ids)).values(last_checked_at=now))
await session.commit()
async def run_due_monitors(app) -> None:
"""Start a check for every enabled Monitor that is due and not already
being checked. Returns once they are started, not finished.
@@ -55,19 +110,23 @@ async def run_due_monitors(app) -> None:
global_interval = int(app.config.get("MONITORS_POLL_INTERVAL", 60))
now = datetime.now(timezone.utc)
in_flight, tasks = _state(app)
memory = _memory(app)
async with app.db_sessionmaker() as session:
monitors = list((await session.execute(
select(Monitor).where(Monitor.enabled.is_(True))
)).scalars())
due = [m for m in monitors if m.id not in in_flight and is_due(m, now, global_interval)]
if not due:
return
# Stamp the scheduled start before checking: the next run is due one
# interval from now, however long this check takes.
await session.execute(
update(Monitor).where(Monitor.id.in_([m.id for m in due])).values(last_checked_at=now))
await session.commit()
monitors = await _enabled_monitors(app)
due = [m for m in monitors if m.id not in in_flight
and is_due(m, now, global_interval, memory.started.get(m.id))]
if not due:
return
# Stamp the scheduled start before checking: the next run is due one
# interval from now, however long this check takes. Kept in memory too,
# for when the database doesn't take it.
for m in due:
memory.started[m.id] = now
try:
await asyncio.wait_for(_stamp(app, [m.id for m in due], now), MONITORS_READ_S)
except Exception as exc:
logger.warning("Monitor starts not stored within %.0f s (%s); kept in memory",
MONITORS_READ_S, exc or type(exc).__name__)
for m in due:
in_flight.add(m.id)
@@ -77,18 +136,31 @@ async def run_due_monitors(app) -> None:
async def _check_and_record(app, monitor: Monitor) -> None:
"""Run one monitor's check and store its result and metrics."""
"""Run one monitor's check, hold its result, and store it."""
in_flight, _ = _state(app)
try:
try:
res = await asyncio.wait_for(run_monitor(monitor), CHECK_DEADLINE_SECONDS)
except asyncio.TimeoutError:
res = dict(_BLANK, error_msg=f"Check did not finish within {CHECK_DEADLINE_SECONDS} s")
check_time = datetime.now(timezone.utc)
async with app.db_sessionmaker() as session:
async with session.begin():
held = holding(app, "Monitor checks", store=lambda rounds: _store(app, rounds),
limit=HELD_RESULTS)
held.hold((monitor.id, monitor.type, monitor.name, datetime.now(timezone.utc), res))
await held.flush(STORE_WAIT_S)
except Exception:
logger.exception("Monitor %r check errored", monitor.name)
finally:
in_flight.discard(monitor.id)
async def _store(app, checks: list) -> None:
"""Write held check results, oldest first, each at the time it was checked."""
async with app.db_sessionmaker() as session:
async with session.begin():
batch = MetricBatch()
for monitor_id, mtype, name, check_time, res in checks:
session.add(MonitorResult(
monitor_id=monitor.id,
monitor_id=monitor_id,
checked_at=check_time,
is_up=res["is_up"],
response_ms=res["response_ms"],
@@ -99,20 +171,9 @@ async def _check_and_record(app, monitor: Monitor) -> None:
error_msg=res["error_msg"],
))
# Alert pipeline — keyed by monitor type (source) + name.
batch = MetricBatch()
batch.add(
source_module=monitor.type,
resource_name=monitor.name, metric_name="is_up",
value=1.0 if res["is_up"] else 0.0,
)
batch.add(source_module=mtype, resource_name=name, metric_name="is_up",
value=1.0 if res["is_up"] else 0.0, recorded_at=check_time)
if res["response_ms"] is not None:
batch.add(
source_module=monitor.type,
resource_name=monitor.name, metric_name="response_ms",
value=res["response_ms"],
)
await batch.record(session)
except Exception:
logger.exception("Monitor %r check errored", monitor.name)
finally:
in_flight.discard(monitor.id)
batch.add(source_module=mtype, resource_name=name, metric_name="response_ms",
value=res["response_ms"], recorded_at=check_time)
await batch.record(session)
tests/core/test_holding.py
+23 -9
View File
@@ -34,15 +34,12 @@ def test_past_its_limit_the_oldest_round_goes_and_is_counted():
assert len(held) == 2 and held.dropped == 1 and [i for _, i in held._held] == ["b", "c"]
def test_a_store_that_fails_or_overruns_its_deadline_keeps_the_rounds_held():
def test_a_store_that_cant_reach_the_database_or_overruns_its_deadline_keeps_the_rounds_held():
async def main():
calls = []
async def unreachable(rounds):
raise ConnectionRefusedError("database down")
async def failing(rounds):
calls.append(list(rounds))
raise RuntimeError("database refused")
held = Holding("t", failing, limit=10)
held = Holding("t", unreachable, limit=10)
held.hold("a")
assert await held.flush(1.0) is False and len(held) == 1
@@ -52,9 +49,26 @@ def test_a_store_that_fails_or_overruns_its_deadline_keeps_the_rounds_held():
stuck = Holding("t", never, limit=10, deadline_s=0.05)
stuck.hold("a")
assert await stuck.flush(1.0) is False and len(stuck) == 1 # given up, not lost
return calls
assert asyncio.run(main()) == [["a"]]
asyncio.run(main())
def test_a_round_the_database_refuses_is_dropped_and_holds_up_nobody_after_it():
async def main():
stored = []
async def store(rounds):
if "bad" in rounds:
raise ValueError("value out of range")
stored.extend(rounds)
held = Holding("t", store, limit=10)
for r in ("a", "bad", "c"):
held.hold(r)
assert await held.flush(1.0) is True
return stored
assert asyncio.run(main()) == ["a", "c"]
def test_an_app_has_one_holding_per_collector():
tests/core/test_monitor_schedule.py
+38
View File
@@ -152,3 +152,41 @@ async def test_nothing_due_stamps_nothing(monkeypatch):
app = _app(store)
await sched.run_due_monitors(app)
assert store.stamped == [] and store.added == []
class _HungSession(_Session):
"""The storage under Postgres has stalled: nothing answers until `gate`."""
def __init__(self, store, gate):
super().__init__(store)
self.gate = gate
async def __aenter__(self):
await self.gate.wait()
return self
async def test_checks_go_on_while_the_database_hangs_and_are_stored_at_their_own_time(monkeypatch):
async def fake_run(m):
return dict(sched._BLANK, is_up=False, error_msg="502 Bad Gateway")
monkeypatch.setattr(sched, "run_monitor", fake_run)
monkeypatch.setattr(sched, "MetricBatch", _Batch)
monkeypatch.setattr(sched, "MONITORS_READ_S", 0.05)
monkeypatch.setattr(sched, "STORE_WAIT_S", 0.05)
store, gate = _Store([_m("app")]), asyncio.Event()
app = NS(config={"MONITORS_POLL_INTERVAL": 60}, db_sessionmaker=lambda: _HungSession(store, gate))
sched._memory(app).monitors = list(store.monitors) # as last read, before the stall
await sched.run_due_monitors(app) # the list and the stamp time out
await _settle(app) # the result is held, the monitor freed
assert store.added == [] and sched._state(app)[0] == set()
await sched.run_due_monitors(app)
await _settle(app)
assert store.added == [] # its start was kept in memory: not due again
checked_by = datetime.now(timezone.utc)
gate.set() # the database answers
await app._holdings["Monitor checks"].flush(1.0)
[res] = store.added
assert res.is_up is False and res.checked_at < checked_by # stored at the time it was checked