diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py index 178c8fb..32863cf 100644 --- a/steward/alerts/routes.py +++ b/steward/alerts/routes.py @@ -7,7 +7,7 @@ from steward.auth.middleware import require_role from steward.alerts import service as alert_svc from steward.core.time_range import parse_range from steward.ansible import sources as src_module -from steward.models.alerts import AlertRule, AlertOperator, MaintenanceWindow +from steward.models.alerts import AlertRule, AlertOperator, AlertState, MaintenanceWindow from steward.models.users import UserRole alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts") @@ -98,8 +98,23 @@ async def list_alerts(): async with current_app.db_sessionmaker() as db: active = await alert_svc.active_rows(db, datetime.now(timezone.utc)) rules = await alert_svc.list_rules(db) + states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()} - return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS) + # Every rule with what it is doing now; disabled rules last. + rules.sort(key=lambda r: (not r.enabled, r.name.lower())) + rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states} + return await render_template( + "alerts/list.html", active=active, rules=rules, rule_state=rule_state, + operators=_OPERATORS, poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60)) + + +@alerts_bp.get("/active") +@require_role(UserRole.viewer) +async def active_fragment(): + """The Active section alone, which the alerts page polls.""" + async with current_app.db_sessionmaker() as db: + active = await alert_svc.active_rows(db, datetime.now(timezone.utc)) + return await render_template("alerts/_active.html", active=active) HISTORY_LIMIT = 200 @@ -298,9 +313,14 @@ async def list_maintenance(): now = datetime.now(timezone.utc) async with current_app.db_sessionmaker() as db: windows = await alert_svc.list_maintenance_windows(db) + # Active first (ending soonest first), then upcoming (starting soonest + # first); expired ones are history, newest first, folded away. + active = sorted((w for w in windows if w.start_at <= now <= w.end_at), key=lambda w: w.end_at) + upcoming = sorted((w for w in windows if w.start_at > now), key=lambda w: w.start_at) + expired = sorted((w for w in windows if w.end_at < now), key=lambda w: w.end_at, reverse=True) return await render_template( "alerts/maintenance.html", - windows=windows, + windows=windows, active=active, upcoming=upcoming, expired=expired, now=now, source_modules=alert_svc.source_modules(), ) diff --git a/steward/alerts/service.py b/steward/alerts/service.py index 0c1ae5c..2811c1c 100644 --- a/steward/alerts/service.py +++ b/steward/alerts/service.py @@ -9,12 +9,12 @@ from __future__ import annotations import logging from datetime import datetime, timezone -from sqlalchemy import or_, select +from sqlalchemy import func, or_, select from steward.models.alerts import ( AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow, ) -from steward.models.metrics import PluginMetric +from steward.models.metrics import PluginMetric, PluginMetricLatest from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor logger = logging.getLogger(__name__) @@ -86,13 +86,13 @@ def alert_label(state: AlertState) -> str: async def latest_value(db, rule: AlertRule) -> float | None: - """The newest sample of the series a rule watches, or None before the first.""" + """The newest sample of the series a rule watches, or None before the first + (a primary-key read of the latest-value table, #5208).""" return (await db.execute( - select(PluginMetric.value) - .where(PluginMetric.source_module == rule.source_module, - PluginMetric.resource_name == rule.resource_name, - PluginMetric.metric_name == rule.metric_name) - .order_by(PluginMetric.recorded_at.desc()).limit(1) + select(PluginMetricLatest.value) + .where(PluginMetricLatest.source_module == rule.source_module, + PluginMetricLatest.resource_name == rule.resource_name, + PluginMetricLatest.metric_name == rule.metric_name) )).scalar_one_or_none() @@ -143,7 +143,10 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None, if label == "no_data": return {"lead": f"No {metric} from ", "resource": rule.resource_name, "tail": f" for {held}" if held else ""} - for_how_long = f", {label} for {held}" if held else f", {label}" + # An acknowledged alert is still breaching: say for how long it has been, + # not how long ago someone acknowledged it (the page says that apart). + word = "breaching" if label == "acknowledged" else label + for_how_long = f", {word} for {held}" if held else f", {word}" if value is None: reading = " has no reading yet" elif metric in _BINARY: @@ -156,18 +159,59 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None, "tail": f"{reading}{condition}{for_how_long}"} +# The order the active list reads in: what is breaching and unhandled, then +# what has gone silent, then what someone has taken, then what may yet clear. +ACTIVE_ORDER = {"firing": 0, "no_data": 1, "acknowledged": 2, "pending": 3} + +_BREACHING = (AlertStateEnum.pending.value, AlertStateEnum.firing.value, + AlertStateEnum.acknowledged.value) + + +async def breach_started(db, rule_ids) -> dict[str, datetime]: + """{rule_id: when its current breach began}: the newest transition into + pending or firing from a quiet state. Acknowledging moves an alert's + last_transition_at, so that column cannot say this.""" + if not rule_ids: + return {} + return dict((await db.execute( + select(AlertEvent.rule_id, func.max(AlertEvent.transitioned_at)) + .where(AlertEvent.rule_id.in_(list(rule_ids)), + AlertEvent.from_state.not_in(_BREACHING), + AlertEvent.to_state.in_((AlertStateEnum.pending.value, AlertStateEnum.firing.value))) + .group_by(AlertEvent.rule_id) + )).all()) + + async def active_rows(db, now: datetime) -> list[dict]: - """The alerts page's active list: each alert's state, rule, sentence and - the link to what it watches.""" + """The alerts page's active list, firing first, then no data, then + acknowledged, then pending; within each the longest-running first. Each + row: its state, rule, label, sentence, the link to what it watches, when + the breach began, and who acknowledged it.""" + from steward.models.users import User + alerts = await active_alerts(db) + started = await breach_started(db, {rule.id for state, rule in alerts + if state.state.value in _BREACHING}) + ackers = {s.acknowledged_by for s, _ in alerts if s.acknowledged_by} + names = dict((await db.execute( + select(User.id, User.username).where(User.id.in_(ackers)))).all()) if ackers else {} rows = [] - for state, rule in await active_alerts(db): + for state, rule in alerts: label = alert_label(state) - since = state.no_data_since if label == "no_data" else state.last_transition_at + if label == "no_data": + since = state.no_data_since + elif label == "acknowledged": + since = started.get(rule.id, state.last_transition_at) + else: + since = state.last_transition_at rows.append({ - "state": state, "rule": rule, "label": label, + "state": state, "rule": rule, "label": label, "since": since, + "breach_since": started.get(rule.id) if label != "no_data" else None, + "acked_by": names.get(state.acknowledged_by, state.acknowledged_by), "says": alert_sentence(rule, label, await latest_value(db, rule), since, now), "url": await resource_url(db, rule), }) + epoch = datetime.max.replace(tzinfo=timezone.utc) + rows.sort(key=lambda r: (ACTIVE_ORDER.get(r["label"], 9), r["since"] or epoch)) return rows diff --git a/steward/ansible/routes.py b/steward/ansible/routes.py index dd291c7..4d6d165 100644 --- a/steward/ansible/routes.py +++ b/steward/ansible/routes.py @@ -443,6 +443,7 @@ async def schedules(): }) if editing_id and s.id == editing_id: editing = s + rows.sort(key=_schedule_order) # Default the source dropdown to the first source (or the edited schedule's # source), and pre-render that source's playbooks so the playbook dropdown @@ -472,6 +473,23 @@ async def schedules(): ) +def _schedule_order(row: dict) -> tuple: + """Schedules list order: failing first (its last run failed or was cut + short, or its last firing could not start one), then running, then the + rest, paused last; by name within each.""" + s, run = row["s"], row["last_run"] + status = run.status.value if run else None + if not s.enabled: + rank = 3 + elif status in ("failed", "interrupted") or (s.last_error and run is None): + rank = 0 + elif status in ("queued", "running"): + rank = 1 + else: + rank = 2 + return rank, s.name.lower() + + def _schedule_form_fields(form) -> tuple[dict | None, str | None]: """Validate + extract schedule fields from a form. Returns (fields, error).""" name = (form.get("name", "") or "").strip() diff --git a/steward/hosts/routes.py b/steward/hosts/routes.py index 9fbf262..0cad8d1 100644 --- a/steward/hosts/routes.py +++ b/steward/hosts/routes.py @@ -50,14 +50,17 @@ async def list_hosts(): hosts = result.scalars().all() host_status = await host_svc.host_status(db, [h.id for h in hosts]) uptime = await host_svc.compute_uptime(db) - agent_metrics = await host_svc.agent_metrics_by_host(db, [h.name for h in hosts]) + agent = await host_svc.agent_overview_by_host( + db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(current_app.config)) + hosts = sorted(hosts, key=lambda h: ( + host_svc.host_rank(host_status.get(h.id), agent.get(h.name)), h.name.lower())) return await render_template( "hosts/list.html", hosts=hosts, host_status=host_status, uptime=uptime, - agent_metrics=agent_metrics, + agent=agent, ) diff --git a/steward/hosts/service.py b/steward/hosts/service.py index f625864..bd16941 100644 --- a/steward/hosts/service.py +++ b/steward/hosts/service.py @@ -113,6 +113,21 @@ async def host_status(db, host_ids: list[str]) -> dict[str, dict]: return out +def host_rank(status: dict | None, agent: dict | None) -> int: + """Where a host sorts in the Hosts list: a down monitor first, then an + agent gone quiet, then monitors not yet checked, then the healthy; a host + with neither monitors nor agent last.""" + if status and status["state"] == "down": + return 0 + if agent and not agent.get("fresh"): + return 1 + if status and status["state"] == "pending": + return 2 + if status or agent: + return 3 + return 4 + + async def host_monitors(db, host_id: str) -> list[dict]: """Display rows for the monitors linked to one host (host hub section).""" monitors = (await db.execute( @@ -134,17 +149,6 @@ async def host_monitors(db, host_id: str) -> list[dict]: return data -async def agent_metrics_by_host(db, host_names: list[str]) -> dict[str, dict]: - """Latest agent cpu/mem per host name, from plugin_metric_latest. - - Core-safe (no host_agent import), so the Hosts list can show an - at-a-glance agent column without importing the host_agent plugin. - """ - from steward.core.metric_series import latest_by_resource - latest = await latest_by_resource(db, "host_agent", host_names, ("cpu_pct", "mem_used_pct")) - return {name: {m: r.value for m, r in metrics.items()} for name, metrics in latest.items()} - - async def agent_overview_by_host( db, host_names: list[str], stale_after_seconds: int = 180, ) -> dict[str, dict]: diff --git a/steward/monitors/service.py b/steward/monitors/service.py index 9adc47d..474f55f 100644 --- a/steward/monitors/service.py +++ b/steward/monitors/service.py @@ -152,6 +152,20 @@ def row_state(monitor: Monitor, latest, uptime_pct, thresholds: dict) -> str: return "up" +def down_reason(latest) -> str | None: + """Why a failed check failed, in a few words: the probe's error, else the + HTTP status, else a content mismatch; None when the result says nothing.""" + if latest is None: + return None + if latest.error_msg: + return latest.error_msg + if latest.status_code: + return f"HTTP {latest.status_code}" + if latest.content_matched is False: + return "expected content not found" + return None + + def problems_first(rows: list[dict]) -> list[dict]: """Down, then degraded, then pending, up and paused; by name within each.""" return sorted(rows, key=lambda d: (STATE_ORDER[d["state"]], d["monitor"].name.lower())) @@ -191,7 +205,13 @@ async def monitor_rows(db, range_key: str, thresholds: dict): "title": _heartbeat_title(m.type, r)} for r in rows], "uptime_pct": uptime_pct, "tls_days": tls_days, + "down_since": None, + "down_reason": down_reason(latest) if latest is not None and not latest.is_up else None, }) + # When each down monitor went down (a query per down monitor: few, and indexed). + for d in data: + if d["state"] == "down": + d["down_since"] = await down_since(db, d["monitor"].id) counts = {k: sum(d["state"] == k for d in data) for k in STATE_ORDER} counts["up"] += counts["degraded"] return problems_first(data), counts, range_key diff --git a/steward/templates/alerts/_active.html b/steward/templates/alerts/_active.html new file mode 100644 index 0000000..4233544 --- /dev/null +++ b/steward/templates/alerts/_active.html @@ -0,0 +1,51 @@ +{# alerts/_active.html — the Alerts page's Active section, polled on its own: + firing first, then no data, then acknowledged, then pending + (alert_svc.active_rows). An acknowledged alert says how long it has been + breaching, and apart from that who acknowledged it and when. #} +{% from "_macros.html" import status_badge, empty_state %} +{% set can_edit = session.user_role in ('operator', 'admin') %} +{% if active %} +
| State | +What is wrong | +Rule | ++ |
|---|---|---|---|
| + {{ status_badge(row.label, title=("No sample since " ~ (row.state.last_sample_at.strftime('%Y-%m-%d %H:%M UTC') if row.state.last_sample_at else 'the rule started watching')) if row.label == 'no_data' else none) }} + {% if row.state.no_data_since and row.label != 'no_data' %} + {{ status_badge('no_data', title="Silent since " ~ row.state.no_data_since.strftime('%Y-%m-%d %H:%M UTC')) }} + {% endif %} + | +
+ {{- row.says.lead -}}
+ {%- if row.url %}{{ row.says.resource }}{% else %}{{ row.says.resource }}{% endif -%}
+ {{- row.says.tail -}}
+ {%- if row.label == 'acknowledged' and row.state.acknowledged_at %}
+ Acknowledged{% if row.acked_by %} by {{ row.acked_by }}{% endif %} {{ row.state.acknowledged_at|ago }}
+ {%- endif %}
+ |
+ {{ row.rule.name }} | ++ {% if can_edit and row.state.state.value == 'firing' %} + + {% endif %} + | +
| State | -What is wrong | -Rule | -- |
|---|---|---|---|
| - {{ status_badge(row.label, title=("No sample since " ~ (row.state.last_sample_at.strftime('%Y-%m-%d %H:%M UTC') if row.state.last_sample_at else 'the rule started watching')) if row.label == 'no_data' else none) }} - {% if row.state.no_data_since and row.label != 'no_data' %} - {{ status_badge('no_data', title="Silent since " ~ row.state.no_data_since.strftime('%Y-%m-%d %H:%M UTC')) }} - {% endif %} - | -- {{- row.says.lead -}} - {%- if row.url %}{{ row.says.resource }}{% else %}{{ row.says.resource }}{% endif -%} - {{- row.says.tail -}} - | -{{ row.rule.name }} | -- {% if can_edit and row.state.state.value == 'firing' %} - - {% endif %} - | -
| {{ w.name }} | @@ -41,10 +39,10 @@ | {{ w.end_at.strftime("%Y-%m-%d %H:%M") }} UTC | -- {% if is_active %}{{ status_badge('warn', 'Active') }} - {% elif is_upcoming %}{{ status_badge('pending', 'Upcoming') }} - {% else %}{{ status_badge('inactive', 'Expired') }}{% endif %} + |
+ {% if kind == 'active' %}{{ status_badge('warn', 'Active') }} ends {{ w.end_at|ago }}
+ {% elif kind == 'upcoming' %}{{ status_badge('pending', 'Upcoming') }}starts {{ w.start_at|ago }}
+ {% else %}{{ status_badge('inactive', 'Expired') }}ended {{ w.end_at|ago }} {% endif %}
|
port must be between 1 and 65535, got 0.
' in page assert 'value="db"' in page and 'value="10.0.0.9"' in page and 'value="30"' in page + + +def test_down_reason_says_why_in_a_few_words(): + from steward.monitors.service import down_reason + assert down_reason(_check(False)) == "503 Service Unavailable" # the probe's own error first + assert down_reason(SimpleNamespace(error_msg=None, status_code=502, content_matched=None)) == "HTTP 502" + assert down_reason(SimpleNamespace(error_msg=None, status_code=None, content_matched=False)) == \ + "expected content not found" + assert down_reason(None) is None + + +async def test_a_down_monitor_says_since_when_and_why_in_the_list(app): + row = _row(_mon("api", mid="m2"), "down", _check(False), 90.0) + row.update(down_since=datetime.now(timezone.utc) - timedelta(minutes=12), + down_reason="connection refused") + async with app.test_request_context("/monitors/rows"): + page = await render_template("monitors/rows.html", monitor_data=[row], + counts={"up": 0, "degraded": 0, "down": 1, "pending": 0, "paused": 0}, + range_key="24h") + page = " ".join(page.split()) + assert re.search(r'Down · connection refused', page), page diff --git a/tests/core/test_problems_first.py b/tests/core/test_problems_first.py new file mode 100644 index 0000000..47d84cc --- /dev/null +++ b/tests/core/test_problems_first.py @@ -0,0 +1,31 @@ +"""Lists put what needs looking at first (#5210): hosts by their worst +signal, schedules failing first, maintenance windows active then upcoming.""" +from types import SimpleNamespace as NS + +from steward.ansible.routes import _schedule_order +from steward.hosts.service import host_rank + + +def test_hosts_sort_down_then_quiet_agent_then_pending_then_healthy(): + down, up, pending = {"state": "down"}, {"state": "up"}, {"state": "pending"} + fresh, quiet = {"fresh": True}, {"fresh": False} + assert host_rank(down, fresh) == 0 + assert host_rank(up, quiet) == 1 # monitors fine, but the agent went silent + assert host_rank(None, quiet) == 1 + assert host_rank(pending, fresh) == 2 + assert host_rank(up, fresh) == 3 and host_rank(None, fresh) == 3 + assert host_rank(None, None) == 4 # nothing watches it + + +def _sched(name, *, enabled=True, last_error=None, status=None): + run = NS(status=NS(value=status)) if status else None + return {"s": NS(name=name, enabled=enabled, last_error=last_error), "last_run": run} + + +def test_schedules_sort_failing_first_and_paused_last(): + rows = [_sched("nightly", status="success"), _sched("paused", enabled=False, status="failed"), + _sched("backup", status="failed"), _sched("deploy", status="running"), + _sched("broken", last_error="source not found"), _sched("cut", status="interrupted"), + _sched("busy", last_error="the last run is still going", status="success")] + assert [r["s"].name for r in sorted(rows, key=_schedule_order)] == [ + "backup", "broken", "cut", "deploy", "busy", "nightly", "paused"] diff --git a/tests/integration/test_metric_latest.py b/tests/integration/test_metric_latest.py index 9bc60ce..759a72d 100644 --- a/tests/integration/test_metric_latest.py +++ b/tests/integration/test_metric_latest.py @@ -72,7 +72,7 @@ def test_the_newest_sample_wins_and_a_late_backlog_does_not_move_it_back(app): def test_a_quiet_host_keeps_its_last_values_and_reads_stale_not_missing(app): from steward.core.alerts import record_metrics - from steward.hosts.service import agent_metrics_by_host, agent_overview_by_host + from steward.hosts.service import agent_overview_by_host now = datetime.now(timezone.utc) quiet, live = ("quiet-" + uuid.uuid4().hex[:8]), ("live-" + uuid.uuid4().hex[:8]) @@ -87,18 +87,15 @@ def test_a_quiet_host_keeps_its_last_values_and_reads_stale_not_missing(app): _sample(quiet + ":/", "disk_used_pct", 56.0, two_hours_ago), _sample(live, "cpu_pct", 7.0, now - timedelta(seconds=20)), ]) - overview = await agent_overview_by_host(s, [quiet, live, "never-" + quiet], 180) - metrics = await agent_metrics_by_host(s, [quiet, live]) - return overview, metrics + return await agent_overview_by_host(s, [quiet, live, "never-" + quiet], 180) - overview, metrics = asyncio.run(go()) + overview = asyncio.run(go()) q = overview[quiet] assert (q["cpu_pct"], q["mem_used_pct"], q["disk_root"]) == (12.0, 34.0, 56.0) assert q["fresh"] is False assert abs((q["last_at"] - two_hours_ago).total_seconds()) < 1 assert overview[live]["fresh"] is True assert "never-" + quiet not in overview # no samples at all: absent - assert metrics[quiet] == {"cpu_pct": 12.0, "mem_used_pct": 34.0} def test_retention_drops_a_latest_value_with_its_raw_history(app):