Milestone 495 "Email goes to each account, in its own time zone", on the operator's ruling (System 156): notifications go to each user, times in their zone, defaults on, toggled per user or per role. - users.notify_alerts / notify_report (migration 0036): None follows the role's default, notify.<kind>.<role> in settings, all on to start. - mail.pick_recipients: active accounts that want the email and have an address mail can reach (an SSO placeholder like name@oidc is skipped and flagged), each in its own zone with no UTC beside it, then the Also send to addresses in their chosen zone with UTC beside. One copy per address, all over one SMTP connection. - Alerts, the weekly report and the test email (to you, in your zone, plus the typed addresses) send that way. The save-time smtp.tz capture is gone. - Notifications: a role × email table of defaults, who each email reaches and who can't be reached, and Also send to with its own zone, which starts as the first admin's to open the page. - Account page: your address and your two switches. Users: an Emailed column, and an admin sets an account's switches on its page. - The Alerts page counts the accounts it emails. zone_select macro and formatting.zones_by_region shared by Account and Notifications. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
195 lines
7.9 KiB
Python
195 lines
7.9 KiB
Python
"""steward/core/reports.py
|
|
|
|
Weekly digest report: uptime, active alerts, alert events, top metrics.
|
|
Called by the scheduler (hourly check) or triggered manually from Settings.
|
|
"""
|
|
from __future__ import annotations
|
|
import logging
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import and_, case, func, select
|
|
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.formatting import zone
|
|
from steward.core.mail import message, recipients, report_email, send
|
|
from steward.core.settings import get_setting
|
|
from steward.models.alerts import AlertEvent
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.monitors import Monitor, MonitorResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
async def build_report_data(app) -> dict:
|
|
"""Collect all data for the weekly digest."""
|
|
now = datetime.now(timezone.utc)
|
|
cutoff_7d = now - timedelta(days=7)
|
|
cutoff_24h = now - timedelta(hours=24)
|
|
|
|
async with app.db_sessionmaker() as db:
|
|
# ── Uptime summary (7d) — per enabled monitor of any type ─────────────
|
|
monitors = (await db.execute(
|
|
select(Monitor).where(Monitor.enabled.is_(True)).order_by(Monitor.name)
|
|
)).scalars().all()
|
|
|
|
uptime_rows = await db.execute(
|
|
select(
|
|
MonitorResult.monitor_id,
|
|
func.count(MonitorResult.id).label("total"),
|
|
func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up"),
|
|
)
|
|
.where(MonitorResult.checked_at >= cutoff_7d)
|
|
.group_by(MonitorResult.monitor_id)
|
|
)
|
|
uptime_map: dict[str, float | None] = {}
|
|
for row in uptime_rows:
|
|
pct = round(float(row.up) / float(row.total) * 100, 2) if row.total else None
|
|
uptime_map[row.monitor_id] = pct
|
|
|
|
uptime_summary = [
|
|
{"name": m.name, "pct_7d": uptime_map.get(m.id)} for m in monitors
|
|
]
|
|
|
|
# ── Active alerts ─────────────────────────────────────────────────────
|
|
active_rows = await alert_svc.active_alerts(db)
|
|
active_alerts = [
|
|
{
|
|
"state": alert_svc.alert_label(state),
|
|
"name": rule.name,
|
|
"resource": rule.resource_name,
|
|
"condition": alert_svc.rule_condition(rule),
|
|
}
|
|
for state, rule in active_rows
|
|
]
|
|
|
|
# ── Alert event counts (7d) ───────────────────────────────────────────
|
|
events_result = await db.execute(
|
|
select(
|
|
func.count(case((AlertEvent.to_state == "firing", 1), else_=None)).label("fired"),
|
|
func.count(case((AlertEvent.to_state == "resolved", 1), else_=None)).label("resolved"),
|
|
)
|
|
.where(AlertEvent.transitioned_at >= cutoff_7d)
|
|
)
|
|
event_row = events_result.one()
|
|
alert_events = {
|
|
"fired": int(event_row.fired or 0),
|
|
"resolved": int(event_row.resolved or 0),
|
|
}
|
|
|
|
# ── Top Traefik routers by avg request_rate (last 24h) ────────────────
|
|
top_routers_result = await db.execute(
|
|
select(
|
|
PluginMetric.resource_name,
|
|
func.avg(PluginMetric.value).label("avg_rate"),
|
|
)
|
|
.where(
|
|
and_(
|
|
PluginMetric.source_module == "traefik",
|
|
PluginMetric.metric_name == "request_rate",
|
|
PluginMetric.recorded_at >= cutoff_24h,
|
|
)
|
|
)
|
|
.group_by(PluginMetric.resource_name)
|
|
.order_by(func.avg(PluginMetric.value).desc())
|
|
.limit(10)
|
|
)
|
|
top_routers = [
|
|
{"name": row.resource_name, "avg_rate": float(row.avg_rate)}
|
|
for row in top_routers_result
|
|
]
|
|
|
|
# ── Monitors currently down (latest result is_up = false) ─────────────
|
|
from steward.core.status import _last_n_results
|
|
enabled = (await db.execute(
|
|
select(Monitor.id, Monitor.name).where(Monitor.enabled.is_(True))
|
|
)).all()
|
|
latest = await _last_n_results(db, [m.id for m in enabled], n=1)
|
|
hosts_down = sorted(name for mid, name in enabled
|
|
if latest.get(mid) and not latest[mid][-1].is_up)
|
|
|
|
return {
|
|
"generated_at": now,
|
|
"uptime_summary": uptime_summary,
|
|
"active_alerts": active_alerts,
|
|
"alert_events": alert_events,
|
|
"top_routers": top_routers,
|
|
"hosts_down": hosts_down,
|
|
}
|
|
|
|
|
|
async def send_report(app) -> tuple[bool, str]:
|
|
"""Email the weekly digest to each account that gets it, and the "Also
|
|
send to" addresses, each in its own zone. Returns (success, message)."""
|
|
smtp_cfg = app.config.get("SMTP", {})
|
|
if not smtp_cfg.get("host"):
|
|
return False, "No mail server is set. Set one in Settings → Notifications."
|
|
async with app.db_sessionmaker() as db:
|
|
to = await recipients(db, "report", smtp_cfg)
|
|
if not to:
|
|
return False, ("Nobody gets the weekly report: no account has it on with a real email "
|
|
"address, and there are no Also send to addresses.")
|
|
|
|
try:
|
|
data = await build_report_data(app)
|
|
except Exception as exc:
|
|
logger.exception("Failed to build report data")
|
|
return False, f"Data collection failed: {exc}"
|
|
|
|
try:
|
|
await send(smtp_cfg, [message(smtp_cfg, r.address, *report_email(smtp_cfg, data, r)) for r in to])
|
|
except Exception as exc:
|
|
logger.error("Failed to send weekly report: %s", exc)
|
|
return False, f"Send failed: {exc}"
|
|
logger.info("Weekly report sent to %d address(es)", len(to))
|
|
return True, f"Report sent to {', '.join(r.address for r in to)}."
|
|
|
|
|
|
def report_due(now: datetime, day: int, hour: int, tz) -> bool:
|
|
"""Whether `now` falls in the scheduled hour, read in the schedule's zone."""
|
|
local = now.astimezone(tz)
|
|
return local.weekday() == day and local.hour == hour
|
|
|
|
|
|
def schedule_in(day: int, hour: int, saved_tz, view_tz, now: datetime | None = None) -> tuple[int, int]:
|
|
"""The schedule's (weekday, hour) as a reader in `view_tz` sees it: the next
|
|
time it falls due in `saved_tz`, read on their clock."""
|
|
local = (now or datetime.now(timezone.utc)).astimezone(saved_tz)
|
|
due = (local + timedelta(days=(day - local.weekday()) % 7)).replace(
|
|
hour=hour, minute=0, second=0, microsecond=0)
|
|
seen = due.astimezone(view_tz)
|
|
return seen.weekday(), seen.hour
|
|
|
|
|
|
async def check_and_send(app) -> None:
|
|
"""Hourly scheduler hook: send if day/hour match and not sent recently."""
|
|
from steward.core.settings import set_setting
|
|
|
|
async with app.db_sessionmaker() as db:
|
|
enabled = await get_setting(db, "reports.enabled")
|
|
if not enabled:
|
|
return
|
|
|
|
schedule_day = int(await get_setting(db, "reports.schedule_day") or 6)
|
|
schedule_hour = int(await get_setting(db, "reports.schedule_hour") or 8)
|
|
schedule_tz = zone(await get_setting(db, "reports.schedule_tz"))
|
|
last_sent_str = await get_setting(db, "reports.last_sent_at") or ""
|
|
|
|
now = datetime.now(timezone.utc)
|
|
if not report_due(now, schedule_day, schedule_hour, schedule_tz):
|
|
return
|
|
|
|
if last_sent_str:
|
|
try:
|
|
last_sent = datetime.fromisoformat(last_sent_str)
|
|
if (now - last_sent).total_seconds() < 23 * 3600:
|
|
return # already sent this window
|
|
except ValueError:
|
|
pass
|
|
|
|
ok, msg = await send_report(app)
|
|
if ok:
|
|
async with app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
await set_setting(db, "reports.last_sent_at", now.isoformat())
|
|
else:
|
|
logger.warning("Weekly report send failed: %s", msg)
|