feat: maintenance windows, reports, uptime widget, alert improvements

- Add maintenance window scheduling to suppress alerts during planned
  downtime (model, routes, templates)
- Add weekly email/webhook reports with host summary (core/reports.py,
  Settings → Reports tab)
- Add host uptime history widget with per-check sparkline view
- Expand alert rules: dynamic metric catalog for plugin sources,
  per-rule HTMX field loading, maintenance window awareness
- Register additional dashboard widgets (uptime, SNMP, UniFi, UPS)
- Add Settings → Reports tab configuration

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-23 08:15:30 -04:00
parent 3c70ac56b3
commit ced1eebe1d
17 changed files with 1286 additions and 84 deletions
+22 -1
View File
@@ -5,7 +5,7 @@ import uuid
from datetime import datetime, timezone
from typing import TYPE_CHECKING
from sqlalchemy import select
from sqlalchemy import and_, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from fabledscryer.models.alerts import (
@@ -14,6 +14,7 @@ from fabledscryer.models.alerts import (
AlertRule,
AlertState,
AlertStateEnum,
MaintenanceWindow,
)
from fabledscryer.models.metrics import PluginMetric
@@ -60,6 +61,26 @@ async def record_metric(
session.add(metric)
await session.flush()
# Check active maintenance windows — skip rule evaluation if suppressed
mw_result = await session.execute(
select(MaintenanceWindow).where(
MaintenanceWindow.start_at <= now,
MaintenanceWindow.end_at >= now,
or_(
MaintenanceWindow.scope_source.is_(None),
and_(
MaintenanceWindow.scope_source == source_module,
or_(
MaintenanceWindow.scope_resource.is_(None),
MaintenanceWindow.scope_resource == resource_name,
),
),
),
).limit(1)
)
if mw_result.scalar_one_or_none() is not None:
return # suppressed by maintenance window
# Load matching enabled rules
result = await session.execute(
select(AlertRule).where(
+275
View File
@@ -0,0 +1,275 @@
"""fabledscryer/core/reports.py
Weekly digest report: uptime, active alerts, alert events, top metrics.
Called by the scheduler (hourly check) or triggered manually from Settings.
"""
from __future__ import annotations
import asyncio
import logging
import smtplib
import ssl
from datetime import datetime, timedelta, timezone
from email.message import EmailMessage
from sqlalchemy import and_, case, func, select
from fabledscryer.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertEvent
from fabledscryer.models.hosts import Host
from fabledscryer.models.metrics import PluginMetric
from fabledscryer.models.monitors import PingResult, PingStatus
logger = logging.getLogger(__name__)
_DAYS = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
async def build_report_data(app) -> dict:
"""Collect all data for the weekly digest."""
now = datetime.now(timezone.utc)
cutoff_7d = now - timedelta(days=7)
cutoff_24h = now - timedelta(hours=24)
async with app.db_sessionmaker() as db:
# ── Uptime summary (7d) ───────────────────────────────────────────────
host_result = await db.execute(
select(Host).where(Host.ping_enabled.is_(True)).order_by(Host.name)
)
hosts = host_result.scalars().all()
uptime_rows = await db.execute(
select(
PingResult.host_id,
func.count(PingResult.id).label("total"),
func.sum(case((PingResult.status == PingStatus.up, 1), else_=0)).label("up"),
)
.where(PingResult.probed_at >= cutoff_7d)
.group_by(PingResult.host_id)
)
uptime_map: dict[str, float | None] = {}
for row in uptime_rows:
pct = round(float(row.up) / float(row.total) * 100, 2) if row.total else None
uptime_map[row.host_id] = pct
uptime_summary = []
for h in hosts:
uptime_summary.append({
"name": h.name,
"pct_7d": uptime_map.get(h.id),
})
# ── Active alerts ─────────────────────────────────────────────────────
active_result = await db.execute(
select(AlertState, AlertRule)
.join(AlertRule, AlertState.rule_id == AlertRule.id)
.where(AlertState.state.in_([
AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending,
]))
.order_by(AlertState.last_transition_at.desc())
)
active_alerts = [
{
"state": state.state.value.upper(),
"name": rule.name,
"resource": rule.resource_name,
"metric": rule.metric_name,
"operator": rule.operator.value,
"threshold": rule.threshold,
}
for state, rule in active_result.all()
]
# ── Alert event counts (7d) ───────────────────────────────────────────
events_result = await db.execute(
select(
func.count(case((AlertEvent.to_state == "firing", 1), else_=None)).label("fired"),
func.count(case((AlertEvent.to_state == "resolved", 1), else_=None)).label("resolved"),
)
.where(AlertEvent.transitioned_at >= cutoff_7d)
)
event_row = events_result.one()
alert_events = {
"fired": int(event_row.fired or 0),
"resolved": int(event_row.resolved or 0),
}
# ── Top Traefik routers by avg request_rate (last 24h) ────────────────
top_routers_result = await db.execute(
select(
PluginMetric.resource_name,
func.avg(PluginMetric.value).label("avg_rate"),
)
.where(
and_(
PluginMetric.source_module == "traefik",
PluginMetric.metric_name == "request_rate",
PluginMetric.recorded_at >= cutoff_24h,
)
)
.group_by(PluginMetric.resource_name)
.order_by(func.avg(PluginMetric.value).desc())
.limit(10)
)
top_routers = [
{"name": row.resource_name, "avg_rate": float(row.avg_rate)}
for row in top_routers_result
]
# ── Hosts currently down ──────────────────────────────────────────────
latest_ping_subq = (
select(PingResult.host_id, func.max(PingResult.probed_at).label("max_at"))
.group_by(PingResult.host_id)
.subquery()
)
down_result = await db.execute(
select(Host.name)
.join(PingResult, PingResult.host_id == Host.id)
.join(
latest_ping_subq,
and_(
PingResult.host_id == latest_ping_subq.c.host_id,
PingResult.probed_at == latest_ping_subq.c.max_at,
),
)
.where(
Host.ping_enabled.is_(True),
PingResult.status == PingStatus.down,
)
.order_by(Host.name)
)
hosts_down = [row.name for row in down_result]
return {
"generated_at": now,
"uptime_summary": uptime_summary,
"active_alerts": active_alerts,
"alert_events": alert_events,
"top_routers": top_routers,
"hosts_down": hosts_down,
}
def _render_report_text(data: dict) -> str:
now = data["generated_at"]
lines: list[str] = []
lines.append("Fabled Scryer — Weekly Report")
lines.append(f"Generated: {now.strftime('%Y-%m-%d %H:%M UTC')}")
lines.append("")
# ── Hosts currently down ──────────────────────────────────────────────────
down = data["hosts_down"]
if down:
lines.append(f"{len(down)} host(s) currently DOWN: {', '.join(down)}")
lines.append("")
# ── Uptime summary ────────────────────────────────────────────────────────
lines.append("UPTIME SUMMARY (7-day)")
lines.append("" * 40)
for entry in data["uptime_summary"]:
pct = entry["pct_7d"]
pct_str = f"{pct:.2f}%" if pct is not None else "no data"
lines.append(f" {entry['name']:<32} {pct_str}")
if not data["uptime_summary"]:
lines.append(" (no ping-enabled hosts)")
lines.append("")
# ── Active alerts ─────────────────────────────────────────────────────────
alerts = data["active_alerts"]
lines.append(f"ACTIVE ALERTS ({len(alerts)})")
lines.append("" * 40)
if alerts:
for a in alerts:
lines.append(
f" [{a['state']}] {a['name']}{a['resource']}:"
f"{a['metric']} {a['operator']} {a['threshold']}"
)
else:
lines.append(" None — all clear.")
lines.append("")
# ── Alert events this week ────────────────────────────────────────────────
ev = data["alert_events"]
lines.append("ALERT EVENTS THIS WEEK")
lines.append("" * 40)
lines.append(f" Fired: {ev['fired']}")
lines.append(f" Resolved: {ev['resolved']}")
lines.append("")
# ── Top Traefik routers ───────────────────────────────────────────────────
if data["top_routers"]:
lines.append("TOP TRAEFIK ROUTERS (avg req/s, last 24h)")
lines.append("" * 40)
for r in data["top_routers"]:
lines.append(f" {r['name']:<40} {r['avg_rate']:.2f} req/s")
lines.append("")
lines.append("" * 40)
lines.append("Fabled Scryer — https://git.fabledsword.com/bvandeusen/FabledScryer")
return "\n".join(lines)
async def send_report(app) -> tuple[bool, str]:
"""Build and email the weekly digest. Returns (success, message)."""
smtp_cfg = app.config.get("SMTP", {})
if not smtp_cfg.get("host") or not smtp_cfg.get("recipients"):
return False, "SMTP not configured — set host and recipients in Settings → Notifications."
try:
data = await build_report_data(app)
except Exception as exc:
logger.exception("Failed to build report data")
return False, f"Data collection failed: {exc}"
body = _render_report_text(data)
date_str = data["generated_at"].strftime("%Y-%m-%d")
subject = f"[Fabled Scryer] Weekly Report — {date_str}"
msg = EmailMessage()
msg["Subject"] = subject
msg["From"] = smtp_cfg.get("username", "fabledscryer@localhost")
msg["To"] = ", ".join(smtp_cfg["recipients"])
msg.set_content(body)
try:
loop = asyncio.get_running_loop()
from fabledscryer.core.notifications import _smtp_send_sync
await loop.run_in_executor(None, _smtp_send_sync, smtp_cfg, msg)
logger.info("Weekly report sent to %s", smtp_cfg["recipients"])
return True, f"Report sent to {', '.join(smtp_cfg['recipients'])}."
except Exception as exc:
logger.error("Failed to send weekly report: %s", exc)
return False, f"Send failed: {exc}"
async def check_and_send(app) -> None:
"""Hourly scheduler hook: send if day/hour match and not sent recently."""
from fabledscryer.core.settings import get_setting, set_setting
async with app.db_sessionmaker() as db:
enabled = await get_setting(db, "reports.enabled")
if not enabled:
return
schedule_day = int(await get_setting(db, "reports.schedule_day") or 6)
schedule_hour = int(await get_setting(db, "reports.schedule_hour") or 8)
last_sent_str = await get_setting(db, "reports.last_sent_at") or ""
now = datetime.now(timezone.utc)
if now.weekday() != schedule_day or now.hour != schedule_hour:
return
if last_sent_str:
try:
last_sent = datetime.fromisoformat(last_sent_str)
if (now - last_sent).total_seconds() < 23 * 3600:
return # already sent this window
except ValueError:
pass
ok, msg = await send_report(app)
if ok:
async with app.db_sessionmaker() as db:
async with db.begin():
await set_setting(db, "reports.last_sent_at", now.isoformat())
else:
logger.warning("Weekly report send failed: %s", msg)
+244 -22
View File
@@ -1,14 +1,14 @@
# fabledscryer/core/widgets.py
#
# Central widget registry. Each entry describes one widget type that can be
# placed on a dashboard. Plugin widgets declare which plugin must be enabled
# for the widget to be available.
# Central widget registry. Each entry describes one widget type that can be
# placed on a dashboard. Plugin widgets declare which plugin must be enabled.
#
# Forward-compatibility notes:
# - Step 8 (widget variants) will add multiple entries per plugin and a
# "variant" sub-key rather than one entry per plugin.
# - Step 9 (plugin management) will allow external plugins to register
# entries here at load time via register_widget().
# "params" is a list of configurable parameters for the widget. Each param:
# key - form field name (stored in config_json)
# label - human-readable label
# type - "select" (only type currently supported)
# default - default value (int or str determines how it's parsed from the form)
# options - list of {value, label} dicts
from __future__ import annotations
WIDGET_REGISTRY: dict[str, dict] = {
@@ -18,8 +18,19 @@ WIDGET_REGISTRY: dict[str, dict] = {
"description": "Live ping status and latency history for all monitored hosts",
"hx_url": "/ping/rows",
"detail_url": "/ping/",
"plugin": None, # built-in; always available
"plugin": None,
"poll": True,
"params": [],
},
"uptime_summary": {
"key": "uptime_summary",
"label": "Uptime / SLA",
"description": "Per-host rolling uptime % for 24h, 7d, and 30d windows",
"hx_url": "/hosts/uptime/widget",
"detail_url": "/hosts/uptime",
"plugin": None,
"poll": False,
"params": [],
},
"dns": {
"key": "dns",
@@ -29,39 +40,250 @@ WIDGET_REGISTRY: dict[str, dict] = {
"detail_url": "/dns/",
"plugin": None,
"poll": True,
"params": [],
},
"traefik": {
"key": "traefik",
"label": "Traefik",
"description": "Proxy request rates, service health, and recent errors",
"traefik_routers": {
"key": "traefik_routers",
"label": "Traefik — Routers",
"description": "Top routers by request rate with error highlights and expiring certs",
"hx_url": "/plugins/traefik/widget",
"detail_url": "/plugins/traefik/",
"plugin": "traefik",
"poll": True,
"params": [],
},
"unifi": {
"key": "unifi",
"label": "UniFi Network",
"description": "WAN status, client counts, active alarms",
"traefik_access_log": {
"key": "traefik_access_log",
"label": "Traefik — Access Log",
"description": "Traffic origin summary — top IPs, countries, and request distribution",
"hx_url": "/plugins/traefik/widget/access_log",
"detail_url": "/plugins/traefik/",
"plugin": "traefik",
"poll": True,
"params": [
{
"key": "ip_filter",
"label": "IP Filter",
"type": "select",
"default": "all",
"options": [
{"value": "all", "label": "All traffic"},
{"value": "internal", "label": "Internal only"},
{"value": "external", "label": "External only"},
],
},
{
"key": "limit",
"label": "Top IPs shown",
"type": "select",
"default": 10,
"options": [
{"value": 10, "label": "10 entries"},
{"value": 20, "label": "20 entries"},
{"value": 50, "label": "50 entries"},
],
},
],
},
"traefik_request_chart": {
"key": "traefik_request_chart",
"label": "Traefik — Request Chart",
"description": "Request rate and error percentage over time (Chart.js)",
"hx_url": "/plugins/traefik/widget/request_chart",
"detail_url": "/plugins/traefik/",
"plugin": "traefik",
"poll": True,
"params": [
{
"key": "hours",
"label": "Time range",
"type": "select",
"default": 6,
"options": [
{"value": 1, "label": "Last 1 hour"},
{"value": 6, "label": "Last 6 hours"},
{"value": 24, "label": "Last 24 hours"},
],
},
],
},
"unifi_overview": {
"key": "unifi_overview",
"label": "UniFi — Overview",
"description": "WAN status, client counts, offline devices, and active alarms",
"hx_url": "/plugins/unifi/widget",
"detail_url": "/plugins/unifi/",
"plugin": "unifi",
"poll": True,
"params": [],
},
"ups": {
"key": "ups",
"label": "UPS",
"description": "Battery charge, runtime estimate, load percentage",
"unifi_clients": {
"key": "unifi_clients",
"label": "UniFi — Top Clients",
"description": "Active wireless and wired clients sorted by bandwidth",
"hx_url": "/plugins/unifi/widget/clients",
"detail_url": "/plugins/unifi/",
"plugin": "unifi",
"poll": True,
"params": [
{
"key": "limit",
"label": "Clients shown",
"type": "select",
"default": 5,
"options": [
{"value": 5, "label": "5 clients"},
{"value": 10, "label": "10 clients"},
{"value": 20, "label": "20 clients"},
],
},
],
},
"unifi_devices": {
"key": "unifi_devices",
"label": "UniFi — Devices",
"description": "Network infrastructure devices with online/offline state",
"hx_url": "/plugins/unifi/widget/devices",
"detail_url": "/plugins/unifi/",
"plugin": "unifi",
"poll": True,
"params": [
{
"key": "type_filter",
"label": "Device type",
"type": "select",
"default": "all",
"options": [
{"value": "all", "label": "All devices"},
{"value": "wireless", "label": "Wireless APs"},
{"value": "wired", "label": "Wired only"},
{"value": "offline", "label": "Offline devices"},
],
},
],
},
"ups_status": {
"key": "ups_status",
"label": "UPS — Status",
"description": "Battery charge, runtime estimate, load percentage, and on-battery alerts",
"hx_url": "/plugins/ups/widget",
"detail_url": "/plugins/ups/",
"plugin": "ups",
"poll": True,
"params": [],
},
"ups_history": {
"key": "ups_history",
"label": "UPS — History Chart",
"description": "Battery %, load %, and input voltage over time (Chart.js)",
"hx_url": "/plugins/ups/widget/history",
"detail_url": "/plugins/ups/",
"plugin": "ups",
"poll": True,
"params": [
{
"key": "hours",
"label": "Time range",
"type": "select",
"default": 6,
"options": [
{"value": 1, "label": "Last 1 hour"},
{"value": 6, "label": "Last 6 hours"},
{"value": 24, "label": "Last 24 hours"},
],
},
],
},
"docker_containers": {
"key": "docker_containers",
"label": "Docker — Containers",
"description": "Container status overview — running/stopped counts and per-container CPU",
"hx_url": "/plugins/docker/widget",
"detail_url": "/plugins/docker/",
"plugin": "docker",
"poll": True,
"params": [
{
"key": "show_stopped",
"label": "Show stopped",
"type": "select",
"default": "no",
"options": [
{"value": "no", "label": "Running only"},
{"value": "yes", "label": "All containers"},
],
},
],
},
"http_monitors": {
"key": "http_monitors",
"label": "HTTP Monitors",
"description": "Synthetic endpoint checks — up/down status and response times",
"hx_url": "/plugins/http/widget",
"detail_url": "/plugins/http/",
"plugin": "http",
"poll": True,
"params": [
{
"key": "show_down_only",
"label": "Display filter",
"type": "select",
"default": "no",
"options": [
{"value": "no", "label": "All monitors"},
{"value": "yes", "label": "Failing only"},
],
},
{
"key": "limit",
"label": "Max shown",
"type": "select",
"default": 10,
"options": [
{"value": 5, "label": "5 monitors"},
{"value": 10, "label": "10 monitors"},
{"value": 20, "label": "20 monitors"},
],
},
],
},
"docker_resources": {
"key": "docker_resources",
"label": "Docker — Resources",
"description": "CPU and memory usage bars for running containers",
"hx_url": "/plugins/docker/widget/resources",
"detail_url": "/plugins/docker/",
"plugin": "docker",
"poll": True,
"params": [
{
"key": "limit",
"label": "Containers shown",
"type": "select",
"default": 10,
"options": [
{"value": 5, "label": "5 containers"},
{"value": 10, "label": "10 containers"},
{"value": 20, "label": "20 containers"},
],
},
],
},
"snmp_devices": {
"key": "snmp_devices",
"label": "SNMP — Devices",
"description": "Latest OID readings for all configured SNMP devices",
"hx_url": "/plugins/snmp/widget",
"detail_url": "/plugins/snmp/",
"plugin": "snmp",
"poll": True,
"params": [],
},
}
def register_widget(definition: dict) -> None:
"""Register a widget at runtime (used by external plugins in step 9)."""
"""Register a widget at runtime (used by external plugins)."""
WIDGET_REGISTRY[definition["key"]] = definition