The last page of #5214's walk. Settings still described Steward as it was before monitors were unified and SNMP was rebuilt, drew its tabs below the breadcrumb, and showed the report schedule in UTC. - Tabs (#5377): settings/_tabs.html is a settings_tabs() macro called in each page's section_tabs block, above the trail, as Ansible's are. The breadcrumb guard also fails any page that {% include %}s a tab strip. - Report time (#5378): reports.schedule_tz keeps the zone of whoever saved the schedule; check_and_send compares in it (report_due), and the form shows the day and hour on the reader's clock (schedule_in) under their zone's name. "Last sent" uses |when. - Wording (#5379): the report contents list what the report holds (monitors, not ping-enabled hosts); "Ping latency" is "Response time" and says what it grades; plugin detail says enabling loads now and disabling needs a restart; the monitor interval says it applies on the next check (the scheduler reads it every pass); the dead list-config branch is gone. - Traefik (#5380): the 5xx and 4xx cutoffs are settings (traefik.threshold.error_5xx_pct 0.1, error_4xx_pct 1.0) in a "Traefik routers" group on Thresholds while the plugin runs, read by router_state through error_cutoffs(). The Traefik page has a Thresholds button for admins, to /settings/thresholds/#traefik. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
266 lines
11 KiB
Python
266 lines
11 KiB
Python
"""steward/core/reports.py
|
|
|
|
Weekly digest report: uptime, active alerts, alert events, top metrics.
|
|
Called by the scheduler (hourly check) or triggered manually from Settings.
|
|
"""
|
|
from __future__ import annotations
|
|
import asyncio
|
|
import logging
|
|
from datetime import datetime, timedelta, timezone
|
|
from email.message import EmailMessage
|
|
|
|
from sqlalchemy import and_, case, func, select
|
|
|
|
from steward.alerts import service as alert_svc
|
|
from steward.models.alerts import AlertEvent
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.monitors import Monitor, MonitorResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_DAYS = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
|
|
|
|
|
|
async def build_report_data(app) -> dict:
|
|
"""Collect all data for the weekly digest."""
|
|
now = datetime.now(timezone.utc)
|
|
cutoff_7d = now - timedelta(days=7)
|
|
cutoff_24h = now - timedelta(hours=24)
|
|
|
|
async with app.db_sessionmaker() as db:
|
|
# ── Uptime summary (7d) — per enabled monitor of any type ─────────────
|
|
monitors = (await db.execute(
|
|
select(Monitor).where(Monitor.enabled.is_(True)).order_by(Monitor.name)
|
|
)).scalars().all()
|
|
|
|
uptime_rows = await db.execute(
|
|
select(
|
|
MonitorResult.monitor_id,
|
|
func.count(MonitorResult.id).label("total"),
|
|
func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up"),
|
|
)
|
|
.where(MonitorResult.checked_at >= cutoff_7d)
|
|
.group_by(MonitorResult.monitor_id)
|
|
)
|
|
uptime_map: dict[str, float | None] = {}
|
|
for row in uptime_rows:
|
|
pct = round(float(row.up) / float(row.total) * 100, 2) if row.total else None
|
|
uptime_map[row.monitor_id] = pct
|
|
|
|
uptime_summary = [
|
|
{"name": m.name, "pct_7d": uptime_map.get(m.id)} for m in monitors
|
|
]
|
|
|
|
# ── Active alerts ─────────────────────────────────────────────────────
|
|
active_rows = await alert_svc.active_alerts(db)
|
|
active_alerts = [
|
|
{
|
|
"state": alert_svc.alert_label(state).upper(),
|
|
"name": rule.name,
|
|
"resource": rule.resource_name,
|
|
"metric": rule.metric_name,
|
|
"operator": rule.operator.value,
|
|
"threshold": rule.threshold,
|
|
}
|
|
for state, rule in active_rows
|
|
]
|
|
|
|
# ── Alert event counts (7d) ───────────────────────────────────────────
|
|
events_result = await db.execute(
|
|
select(
|
|
func.count(case((AlertEvent.to_state == "firing", 1), else_=None)).label("fired"),
|
|
func.count(case((AlertEvent.to_state == "resolved", 1), else_=None)).label("resolved"),
|
|
)
|
|
.where(AlertEvent.transitioned_at >= cutoff_7d)
|
|
)
|
|
event_row = events_result.one()
|
|
alert_events = {
|
|
"fired": int(event_row.fired or 0),
|
|
"resolved": int(event_row.resolved or 0),
|
|
}
|
|
|
|
# ── Top Traefik routers by avg request_rate (last 24h) ────────────────
|
|
top_routers_result = await db.execute(
|
|
select(
|
|
PluginMetric.resource_name,
|
|
func.avg(PluginMetric.value).label("avg_rate"),
|
|
)
|
|
.where(
|
|
and_(
|
|
PluginMetric.source_module == "traefik",
|
|
PluginMetric.metric_name == "request_rate",
|
|
PluginMetric.recorded_at >= cutoff_24h,
|
|
)
|
|
)
|
|
.group_by(PluginMetric.resource_name)
|
|
.order_by(func.avg(PluginMetric.value).desc())
|
|
.limit(10)
|
|
)
|
|
top_routers = [
|
|
{"name": row.resource_name, "avg_rate": float(row.avg_rate)}
|
|
for row in top_routers_result
|
|
]
|
|
|
|
# ── Monitors currently down (latest result is_up = false) ─────────────
|
|
from steward.core.status import _last_n_results
|
|
enabled = (await db.execute(
|
|
select(Monitor.id, Monitor.name).where(Monitor.enabled.is_(True))
|
|
)).all()
|
|
latest = await _last_n_results(db, [m.id for m in enabled], n=1)
|
|
hosts_down = sorted(name for mid, name in enabled
|
|
if latest.get(mid) and not latest[mid][-1].is_up)
|
|
|
|
return {
|
|
"generated_at": now,
|
|
"uptime_summary": uptime_summary,
|
|
"active_alerts": active_alerts,
|
|
"alert_events": alert_events,
|
|
"top_routers": top_routers,
|
|
"hosts_down": hosts_down,
|
|
}
|
|
|
|
|
|
def _render_report_text(data: dict) -> str:
|
|
now = data["generated_at"]
|
|
lines: list[str] = []
|
|
|
|
lines.append("Steward — Weekly Report")
|
|
lines.append(f"Generated: {now.strftime('%Y-%m-%d %H:%M UTC')}")
|
|
lines.append("")
|
|
|
|
# ── Hosts currently down ──────────────────────────────────────────────────
|
|
down = data["hosts_down"]
|
|
if down:
|
|
lines.append(f"⚠ {len(down)} monitor(s) currently DOWN: {', '.join(down)}")
|
|
lines.append("")
|
|
|
|
# ── Uptime summary ────────────────────────────────────────────────────────
|
|
lines.append("UPTIME SUMMARY (7-day)")
|
|
lines.append("─" * 40)
|
|
for entry in data["uptime_summary"]:
|
|
pct = entry["pct_7d"]
|
|
pct_str = f"{pct:.2f}%" if pct is not None else "no data"
|
|
lines.append(f" {entry['name']:<32} {pct_str}")
|
|
if not data["uptime_summary"]:
|
|
lines.append(" (no monitors configured)")
|
|
lines.append("")
|
|
|
|
# ── Active alerts ─────────────────────────────────────────────────────────
|
|
alerts = data["active_alerts"]
|
|
lines.append(f"ACTIVE ALERTS ({len(alerts)})")
|
|
lines.append("─" * 40)
|
|
if alerts:
|
|
for a in alerts:
|
|
lines.append(
|
|
f" [{a['state']}] {a['name']} — {a['resource']}:"
|
|
f"{a['metric']} {a['operator']} {a['threshold']}"
|
|
)
|
|
else:
|
|
lines.append(" None — all clear.")
|
|
lines.append("")
|
|
|
|
# ── Alert events this week ────────────────────────────────────────────────
|
|
ev = data["alert_events"]
|
|
lines.append("ALERT EVENTS THIS WEEK")
|
|
lines.append("─" * 40)
|
|
lines.append(f" Fired: {ev['fired']}")
|
|
lines.append(f" Resolved: {ev['resolved']}")
|
|
lines.append("")
|
|
|
|
# ── Top Traefik routers ───────────────────────────────────────────────────
|
|
if data["top_routers"]:
|
|
lines.append("TOP TRAEFIK ROUTERS (avg req/s, last 24h)")
|
|
lines.append("─" * 40)
|
|
for r in data["top_routers"]:
|
|
lines.append(f" {r['name']:<40} {r['avg_rate']:.2f} req/s")
|
|
lines.append("")
|
|
|
|
lines.append("─" * 40)
|
|
lines.append("Steward — https://git.fabledsword.com/bvandeusen/Steward")
|
|
return "\n".join(lines)
|
|
|
|
|
|
async def send_report(app) -> tuple[bool, str]:
|
|
"""Build and email the weekly digest. Returns (success, message)."""
|
|
smtp_cfg = app.config.get("SMTP", {})
|
|
if not smtp_cfg.get("host") or not smtp_cfg.get("recipients"):
|
|
return False, "SMTP not configured — set host and recipients in Settings → Notifications."
|
|
|
|
try:
|
|
data = await build_report_data(app)
|
|
except Exception as exc:
|
|
logger.exception("Failed to build report data")
|
|
return False, f"Data collection failed: {exc}"
|
|
|
|
body = _render_report_text(data)
|
|
date_str = data["generated_at"].strftime("%Y-%m-%d")
|
|
subject = f"[Steward] Weekly Report — {date_str}"
|
|
|
|
msg = EmailMessage()
|
|
msg["Subject"] = subject
|
|
msg["From"] = smtp_cfg.get("username", "steward@localhost")
|
|
msg["To"] = ", ".join(smtp_cfg["recipients"])
|
|
msg.set_content(body)
|
|
|
|
try:
|
|
loop = asyncio.get_running_loop()
|
|
from steward.core.notifications import _smtp_send_sync
|
|
await loop.run_in_executor(None, _smtp_send_sync, smtp_cfg, msg)
|
|
logger.info("Weekly report sent to %s", smtp_cfg["recipients"])
|
|
return True, f"Report sent to {', '.join(smtp_cfg['recipients'])}."
|
|
except Exception as exc:
|
|
logger.error("Failed to send weekly report: %s", exc)
|
|
return False, f"Send failed: {exc}"
|
|
|
|
|
|
def report_due(now: datetime, day: int, hour: int, tz) -> bool:
|
|
"""Whether `now` falls in the scheduled hour, read in the schedule's zone."""
|
|
local = now.astimezone(tz)
|
|
return local.weekday() == day and local.hour == hour
|
|
|
|
|
|
def schedule_in(day: int, hour: int, saved_tz, view_tz, now: datetime | None = None) -> tuple[int, int]:
|
|
"""The schedule's (weekday, hour) as a reader in `view_tz` sees it: the next
|
|
time it falls due in `saved_tz`, read on their clock."""
|
|
local = (now or datetime.now(timezone.utc)).astimezone(saved_tz)
|
|
due = (local + timedelta(days=(day - local.weekday()) % 7)).replace(
|
|
hour=hour, minute=0, second=0, microsecond=0)
|
|
seen = due.astimezone(view_tz)
|
|
return seen.weekday(), seen.hour
|
|
|
|
|
|
async def check_and_send(app) -> None:
|
|
"""Hourly scheduler hook: send if day/hour match and not sent recently."""
|
|
from steward.core.formatting import zone
|
|
from steward.core.settings import get_setting, set_setting
|
|
|
|
async with app.db_sessionmaker() as db:
|
|
enabled = await get_setting(db, "reports.enabled")
|
|
if not enabled:
|
|
return
|
|
|
|
schedule_day = int(await get_setting(db, "reports.schedule_day") or 6)
|
|
schedule_hour = int(await get_setting(db, "reports.schedule_hour") or 8)
|
|
schedule_tz = zone(await get_setting(db, "reports.schedule_tz"))
|
|
last_sent_str = await get_setting(db, "reports.last_sent_at") or ""
|
|
|
|
now = datetime.now(timezone.utc)
|
|
if not report_due(now, schedule_day, schedule_hour, schedule_tz):
|
|
return
|
|
|
|
if last_sent_str:
|
|
try:
|
|
last_sent = datetime.fromisoformat(last_sent_str)
|
|
if (now - last_sent).total_seconds() < 23 * 3600:
|
|
return # already sent this window
|
|
except ValueError:
|
|
pass
|
|
|
|
ok, msg = await send_report(app)
|
|
if ok:
|
|
async with app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
await set_setting(db, "reports.last_sent_at", now.isoformat())
|
|
else:
|
|
logger.warning("Weekly report send failed: %s", msg)
|