Files
FabledSteward/steward/core/reports.py
T
bvandeusen 35f658b573
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 8s
CI / integration (push) Successful in 2m17s
CI / publish (push) Successful in 1m10s
feat(monitors): unify ping/dns/http into one Monitor entity + custom targets
Collapse the three former check types into a single core `Monitor` entity
with one management surface (/monitors), one result table (monitor_results),
and a single scheduled task. Every type can now watch a free-standing custom
destination (optional host_id) — not just a registered Host.

- models: Monitor + MonitorResult replace PingResult/DnsResult; Host loses its
  ping/dns facet columns (now Monitor rows linked by host_id).
- checks: monitors/{ping,dns,http}.py pure probes + runner.run_monitor
  dispatcher; one monitor_check scheduler with a per-monitor due-filter.
- status: single monitor_status_source replaces the three sources.
- UI: /monitors blueprint (type-aware add/edit/list/widget); host hub shows a
  host's linked monitors + "add monitor for this host"; nav + widget registry
  + alert metric catalog rewired. http plugin folded into core and removed.
- migration 0022 merges the http branch, data-migrates host facets +
  http_monitors + all three result histories, drops the old tables/columns.

Resolves the per-host ping/dns auto-attach issue (#275): monitors are now
explicit, never auto-added to every host.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
2026-06-18 08:56:13 -04:00

269 lines
11 KiB
Python

"""steward/core/reports.py
Weekly digest report: uptime, active alerts, alert events, top metrics.
Called by the scheduler (hourly check) or triggered manually from Settings.
"""
from __future__ import annotations
import asyncio
import logging
from datetime import datetime, timedelta, timezone
from email.message import EmailMessage
from sqlalchemy import and_, case, func, select
from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertEvent
from steward.models.metrics import PluginMetric
from steward.models.monitors import Monitor, MonitorResult
logger = logging.getLogger(__name__)
_DAYS = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
async def build_report_data(app) -> dict:
"""Collect all data for the weekly digest."""
now = datetime.now(timezone.utc)
cutoff_7d = now - timedelta(days=7)
cutoff_24h = now - timedelta(hours=24)
async with app.db_sessionmaker() as db:
# ── Uptime summary (7d) — per enabled monitor of any type ─────────────
monitors = (await db.execute(
select(Monitor).where(Monitor.enabled.is_(True)).order_by(Monitor.name)
)).scalars().all()
uptime_rows = await db.execute(
select(
MonitorResult.monitor_id,
func.count(MonitorResult.id).label("total"),
func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up"),
)
.where(MonitorResult.checked_at >= cutoff_7d)
.group_by(MonitorResult.monitor_id)
)
uptime_map: dict[str, float | None] = {}
for row in uptime_rows:
pct = round(float(row.up) / float(row.total) * 100, 2) if row.total else None
uptime_map[row.monitor_id] = pct
uptime_summary = [
{"name": m.name, "pct_7d": uptime_map.get(m.id)} for m in monitors
]
# ── Active alerts ─────────────────────────────────────────────────────
active_result = await db.execute(
select(AlertState, AlertRule)
.join(AlertRule, AlertState.rule_id == AlertRule.id)
.where(AlertState.state.in_([
AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending,
]))
.order_by(AlertState.last_transition_at.desc())
)
active_alerts = [
{
"state": state.state.value.upper(),
"name": rule.name,
"resource": rule.resource_name,
"metric": rule.metric_name,
"operator": rule.operator.value,
"threshold": rule.threshold,
}
for state, rule in active_result.all()
]
# ── Alert event counts (7d) ───────────────────────────────────────────
events_result = await db.execute(
select(
func.count(case((AlertEvent.to_state == "firing", 1), else_=None)).label("fired"),
func.count(case((AlertEvent.to_state == "resolved", 1), else_=None)).label("resolved"),
)
.where(AlertEvent.transitioned_at >= cutoff_7d)
)
event_row = events_result.one()
alert_events = {
"fired": int(event_row.fired or 0),
"resolved": int(event_row.resolved or 0),
}
# ── Top Traefik routers by avg request_rate (last 24h) ────────────────
top_routers_result = await db.execute(
select(
PluginMetric.resource_name,
func.avg(PluginMetric.value).label("avg_rate"),
)
.where(
and_(
PluginMetric.source_module == "traefik",
PluginMetric.metric_name == "request_rate",
PluginMetric.recorded_at >= cutoff_24h,
)
)
.group_by(PluginMetric.resource_name)
.order_by(func.avg(PluginMetric.value).desc())
.limit(10)
)
top_routers = [
{"name": row.resource_name, "avg_rate": float(row.avg_rate)}
for row in top_routers_result
]
# ── Monitors currently down (latest result is_up = false) ─────────────
latest_subq = (
select(MonitorResult.monitor_id, func.max(MonitorResult.checked_at).label("max_at"))
.group_by(MonitorResult.monitor_id)
.subquery()
)
down_result = await db.execute(
select(Monitor.name)
.join(MonitorResult, MonitorResult.monitor_id == Monitor.id)
.join(
latest_subq,
and_(
MonitorResult.monitor_id == latest_subq.c.monitor_id,
MonitorResult.checked_at == latest_subq.c.max_at,
),
)
.where(
Monitor.enabled.is_(True),
MonitorResult.is_up.is_(False),
)
.order_by(Monitor.name)
)
hosts_down = [row.name for row in down_result]
return {
"generated_at": now,
"uptime_summary": uptime_summary,
"active_alerts": active_alerts,
"alert_events": alert_events,
"top_routers": top_routers,
"hosts_down": hosts_down,
}
def _render_report_text(data: dict) -> str:
now = data["generated_at"]
lines: list[str] = []
lines.append("Steward — Weekly Report")
lines.append(f"Generated: {now.strftime('%Y-%m-%d %H:%M UTC')}")
lines.append("")
# ── Hosts currently down ──────────────────────────────────────────────────
down = data["hosts_down"]
if down:
lines.append(f"⚠ {len(down)} monitor(s) currently DOWN: {', '.join(down)}")
lines.append("")
# ── Uptime summary ────────────────────────────────────────────────────────
lines.append("UPTIME SUMMARY (7-day)")
lines.append("─" * 40)
for entry in data["uptime_summary"]:
pct = entry["pct_7d"]
pct_str = f"{pct:.2f}%" if pct is not None else "no data"
lines.append(f" {entry['name']:<32} {pct_str}")
if not data["uptime_summary"]:
lines.append(" (no monitors configured)")
lines.append("")
# ── Active alerts ─────────────────────────────────────────────────────────
alerts = data["active_alerts"]
lines.append(f"ACTIVE ALERTS ({len(alerts)})")
lines.append("─" * 40)
if alerts:
for a in alerts:
lines.append(
f" [{a['state']}] {a['name']}{a['resource']}:"
f"{a['metric']} {a['operator']} {a['threshold']}"
)
else:
lines.append(" None — all clear.")
lines.append("")
# ── Alert events this week ────────────────────────────────────────────────
ev = data["alert_events"]
lines.append("ALERT EVENTS THIS WEEK")
lines.append("─" * 40)
lines.append(f" Fired: {ev['fired']}")
lines.append(f" Resolved: {ev['resolved']}")
lines.append("")
# ── Top Traefik routers ───────────────────────────────────────────────────
if data["top_routers"]:
lines.append("TOP TRAEFIK ROUTERS (avg req/s, last 24h)")
lines.append("─" * 40)
for r in data["top_routers"]:
lines.append(f" {r['name']:<40} {r['avg_rate']:.2f} req/s")
lines.append("")
lines.append("─" * 40)
lines.append("Steward — https://git.fabledsword.com/bvandeusen/Steward")
return "\n".join(lines)
async def send_report(app) -> tuple[bool, str]:
"""Build and email the weekly digest. Returns (success, message)."""
smtp_cfg = app.config.get("SMTP", {})
if not smtp_cfg.get("host") or not smtp_cfg.get("recipients"):
return False, "SMTP not configured — set host and recipients in Settings → Notifications."
try:
data = await build_report_data(app)
except Exception as exc:
logger.exception("Failed to build report data")
return False, f"Data collection failed: {exc}"
body = _render_report_text(data)
date_str = data["generated_at"].strftime("%Y-%m-%d")
subject = f"[Steward] Weekly Report — {date_str}"
msg = EmailMessage()
msg["Subject"] = subject
msg["From"] = smtp_cfg.get("username", "steward@localhost")
msg["To"] = ", ".join(smtp_cfg["recipients"])
msg.set_content(body)
try:
loop = asyncio.get_running_loop()
from steward.core.notifications import _smtp_send_sync
await loop.run_in_executor(None, _smtp_send_sync, smtp_cfg, msg)
logger.info("Weekly report sent to %s", smtp_cfg["recipients"])
return True, f"Report sent to {', '.join(smtp_cfg['recipients'])}."
except Exception as exc:
logger.error("Failed to send weekly report: %s", exc)
return False, f"Send failed: {exc}"
async def check_and_send(app) -> None:
"""Hourly scheduler hook: send if day/hour match and not sent recently."""
from steward.core.settings import get_setting, set_setting
async with app.db_sessionmaker() as db:
enabled = await get_setting(db, "reports.enabled")
if not enabled:
return
schedule_day = int(await get_setting(db, "reports.schedule_day") or 6)
schedule_hour = int(await get_setting(db, "reports.schedule_hour") or 8)
last_sent_str = await get_setting(db, "reports.last_sent_at") or ""
now = datetime.now(timezone.utc)
if now.weekday() != schedule_day or now.hour != schedule_hour:
return
if last_sent_str:
try:
last_sent = datetime.fromisoformat(last_sent_str)
if (now - last_sent).total_seconds() < 23 * 3600:
return # already sent this window
except ValueError:
pass
ok, msg = await send_report(app)
if ok:
async with app.db_sessionmaker() as db:
async with db.begin():
await set_setting(db, "reports.last_sent_at", now.isoformat())
else:
logger.warning("Weekly report send failed: %s", msg)