diff --git a/docs/architecture.md b/docs/architecture.md index c0e3b3c..41a3a16 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -125,7 +125,7 @@ What learns a normal, and the floor each departure must clear (Settings → Thre Host metrics learn with `parts=False`: the host learns, its cores (`:core`) don't. Disk space, a monitor that is down, an agent gone quiet and a router's 5xx share are failures and keep their fixed cutoffs (a router's shares are judged only over enough requests). -Pages grade by the normal through `anomalies.graded()` (hosts' `host_grades` and `worst_reading(normals=)`, `docker.service.usage(thresholds=, norms=)`, `monitors.service.latency_grade`): a reading past its cutoff but within its normal reads healthy and says what is usual at this hour. The host, monitor, container and Traefik pages list their anomalies with `_anomaly_card.html`. +Pages grade by the normal through `anomalies.graded()` (hosts' `host_grades` and `worst_reading(normals=)`, `docker.service.usage(thresholds=, norms=)`, `monitors.service.latency_grade`): a reading past its cutoff but within its normal reads healthy and says what is usual at this hour. The host, monitor, container and Traefik pages each carry a one-line `_anomaly_card.html`. It says how many anomalies wait for review and names any that are out of their normal now. Its Review button opens Alerts → Anomalies narrowed to that resource (`?source=&resource=`), where they are reviewed. --- diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index 3b636f0..bcd4598 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -482,7 +482,7 @@ async def container_detail(host_id: str, name: str): resource = docker_svc.container_resource(host.name, name) if host is not None else None anomalies = await anom.views(db, await anom.list_anomalies( db, source="docker", resource=resource, - since=datetime.now(timezone.utc) - timedelta(days=30), limit=20), + since=datetime.now(timezone.utc) - timedelta(days=30), limit=100), links=False) if resource else [] if c is None: diff --git a/plugins/host_agent/templates/_host_vitals.html b/plugins/host_agent/templates/_host_vitals.html index ccd02b8..8b4d1ea 100644 --- a/plugins/host_agent/templates/_host_vitals.html +++ b/plugins/host_agent/templates/_host_vitals.html @@ -21,6 +21,8 @@ {%- if reg.agent_version %}{{ " · " if not stale }}agent v{{ reg.agent_version }}{% endif -%} {%- if clock and clock[0] %} · {{ clock[0] }}{% endif -%} + {# Charts, every mount, interface, disk and temperature sensor. #} + Full metrics →
{{ stat_tile("CPU", pct(cpu), state=(grades.cpu or [status_class(cpu, 'cpu')])[0], hint=sparks.cpu|safe) }} diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html index 0cdd7eb..0dc4598 100644 --- a/plugins/host_agent/templates/panel.html +++ b/plugins/host_agent/templates/panel.html @@ -14,10 +14,9 @@
{% if reporting %} - {# Reporting: live vitals are shown by the vitals strip above; this panel is - the way to the full metrics, with the lifecycle actions folded under - "Manage agent" (#5278). #} - Full metrics → + {# Reporting: live vitals, and the way to the full metrics, are in the + vitals strip at the top of the page; this panel holds what the host has + waiting, with the lifecycle actions folded under "Manage agent" (#5278). #} {# What the host has waiting, from the agent's hourly check (#5765): a needed reboot first, then the updates. #} diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index 3721263..efe7a46 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -410,7 +410,7 @@ async def rows(): phases.mark("window") # The routers' metric anomalies, last 30 days (#5741). anomalies = await anom.views(db, await anom.list_anomalies( - db, source="traefik", since=now - timedelta(days=30), limit=20), links=False) + db, source="traefik", since=now - timedelta(days=30), limit=100), links=False) for r in current: points = history[r["name"]] r["since"], r["since_before"] = (problem_since(points, thresholds) if r["state"] != "ok" diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py index a108e68..3053db2 100644 --- a/steward/alerts/routes.py +++ b/steward/alerts/routes.py @@ -284,15 +284,24 @@ async def alert_history(): @require_role(UserRole.viewer) async def anomalies(): """Metric anomalies (#5740): those to review, newest occurrence first, - then the reviewed and lapsed ones under day dividers.""" + then the reviewed and lapsed ones under day dividers. `source` and + `resource` narrow it to one resource's, its parts included: a resource's + page links here with them.""" + from urllib.parse import urlencode from steward.audit.service import mark_day_groups from steward.core import anomalies as anom from steward.core import formatting as fmt + source = request.args.get("source", "").strip() or None + resource = request.args.get("resource", "").strip() or None + only = {"source": source, "resource": resource} async with current_app.db_sessionmaker() as db: - to_review = await anom.views(db, await anom.list_anomalies(db, (anom.TO_REVIEW,))) + to_review = await anom.views(db, await anom.list_anomalies(db, (anom.TO_REVIEW,), **only)) past = await anom.views(db, await anom.list_anomalies( - db, (anom.NORMAL, anom.REAL, anom.LAPSED), limit=PAGE_SIZE * 4)) + db, (anom.NORMAL, anom.REAL, anom.LAPSED), limit=PAGE_SIZE * 4, **only)) days = int(await get_setting(db, "anomalies.review_days") or 14) + here = "/alerts/anomalies" + if source or resource: + here += "?" + urlencode({k: v for k, v in only.items() if v}) tz = fmt.zone(session.get("tz")) to_review = [{"v": v} for v in to_review] past = [{"v": v} for v in past] @@ -300,7 +309,7 @@ async def anomalies(): mark_day_groups(to_review, tz, at=at) mark_day_groups(past, tz, at=at) return await render_template("alerts/anomalies.html", to_review=to_review, past=past, - review_days=days) + review_days=days, source=source, resource=resource, here=here) @alerts_bp.post("/anomalies//review") diff --git a/steward/hosts/routes.py b/steward/hosts/routes.py index a4c5253..01f3731 100644 --- a/steward/hosts/routes.py +++ b/steward/hosts/routes.py @@ -132,10 +132,11 @@ async def host_detail(host_id: str): linkable_targets = (await db.execute( select(AnsibleTarget).where(AnsibleTarget.host_id.is_(None)) .order_by(AnsibleTarget.name))).scalars().all() - # The host's metric anomalies, its disks and interfaces included (#5740). + # The host's metric anomalies, its disks and interfaces included + # (#5740). The card counts those to review, so the limit is generous. anomalies = await anom.views(db, await anom.list_anomalies( db, source="host_agent", resource=host.name, - since=datetime.now(timezone.utc) - timedelta(days=30), limit=10), links=False) + since=datetime.now(timezone.utc) - timedelta(days=30), limit=100), links=False) return await render_template( "hosts/detail.html", diff --git a/steward/monitors/routes.py b/steward/monitors/routes.py index 94c54e3..0e3f3d5 100644 --- a/steward/monitors/routes.py +++ b/steward/monitors/routes.py @@ -128,7 +128,7 @@ async def detail(monitor_id: str): norm = (await monitor_svc.latency_norms(db, [m])).get(m.id) anomalies = await anom.views(db, await anom.list_anomalies( db, source=m.type, resource=m.name, - since=datetime.now(timezone.utc) - timedelta(days=30), limit=20), links=False) + since=datetime.now(timezone.utc) - timedelta(days=30), limit=100), links=False) host = await db.get(Host, m.host_id) if m.host_id else None thresholds = current_app.config.get("THRESHOLDS", {}) latest = st["latest"] diff --git a/steward/templates/_anomaly_card.html b/steward/templates/_anomaly_card.html index d289fc6..f335c6c 100644 --- a/steward/templates/_anomaly_card.html +++ b/steward/templates/_anomaly_card.html @@ -1,14 +1,31 @@ -{# A resource's metric anomalies (#5740), for its own page: `anomalies` are - anomalies.views() rows, to review first; `back` is this page's path, so a - review returns here. Shown only when there is one. #} -{% from "_anomaly.html" import anomaly_row with context %} +{# A resource's metric anomalies, for its own page: one line saying how many + wait for review and which are out of their normal now, with a button to + Alerts → Anomalies narrowed to this resource, where they are reviewed. + `anomalies` are anomalies.views() rows; `base_resource` is the page's + resource (none for a page covering a whole source, like Traefik). Shown + only while one waits for review or is going on. #} {% if anomalies %} -{% set going = anomalies | selectattr("ongoing") | list %} -
-

- {% if going %}Out of its normal now{% else %}Metric anomalies, last 30 days{% endif %} +{% set waiting = anomalies | selectattr("anomaly.review", "equalto", "to_review") | list %} +{% set going = anomalies | selectattr("ongoing") | rejectattr("anomaly.review", "equalto", "normal") | list %} +{% if waiting or going %} +{% set q = {"source": anomalies[0].anomaly.source_module} %} +{% if base_resource %}{% set _ = q.update({"resource": base_resource}) %}{% endif %} +
+

+ {% if going %}Out of its normal now{% else %}Metric anomalies{% endif %}

- {% for v in anomalies %}{{ anomaly_row(v, back=back, show_resource=v.anomaly.resource_name != base_resource) }}{% endfor %} - + + {%- if going -%} + {%- for v in going -%} + {%- if v.anomaly.resource_name != base_resource %}{{ v.anomaly.resource_name }} {% endif -%} + {{ metric_label(v.anomaly.metric_name) }}{{ ", " if not loop.last }} + {%- endfor -%} + {%- if waiting %} · {% endif -%} + {%- endif -%} + {%- if waiting %}{{ waiting|length }} to review{% endif -%} + + Review
{% endif %} +{% endif %} diff --git a/steward/templates/alerts/anomalies.html b/steward/templates/alerts/anomalies.html index 60ee56f..0183721 100644 --- a/steward/templates/alerts/anomalies.html +++ b/steward/templates/alerts/anomalies.html @@ -11,10 +11,17 @@ {% macro divided(items) %} {% for item in items %} {% if item.opens_group %}

{{ item.group }}

{% endif %} - {{ anomaly_row(item.v, back="/alerts/anomalies") }} + {{ anomaly_row(item.v, back=here|default("/alerts/anomalies"), show_resource=not resource or item.v.anomaly.resource_name != resource) }} {% endfor %} {% endmacro %} +{% if source or resource %} +

+ Only {% if resource %}{{ resource }}{% if source %} ({{ source_label(source) }}){% endif %}{% else %}{{ source_label(source) }}{% endif %}. + Show every anomaly +

+{% endif %} +

{% if to_review %}{{ to_review|length }} to review{% else %}Nothing to review{% endif %} diff --git a/tests/core/test_anomalies.py b/tests/core/test_anomalies.py index 004d786..08bb1e9 100644 --- a/tests/core/test_anomalies.py +++ b/tests/core/test_anomalies.py @@ -88,6 +88,45 @@ async def test_a_quiet_page_says_nothing_waits(app): assert "Nothing to review" in page and "within 14 days" in page +async def test_a_page_narrowed_to_one_resource_says_so_and_a_review_returns_to_it(app): + here = "/alerts/anomalies?source=host_agent&resource=nas" + async with app.test_request_context(here): + session["user_id"], session["user_role"] = "u1", "operator" + page = await render_template("alerts/anomalies.html", review_days=14, past=[], + to_review=[{"v": _view(), "opens_group": True, "group": "Today"}], + source="host_agent", resource="nas", here=here) + assert "nas" in page and 'Show every anomaly' in page + assert 'name="next" value="/alerts/anomalies?source=host_agent&resource=nas"' in page + + +async def _card(app, anomalies, base_resource="nas"): + async with app.test_request_context("/hosts/h1"): + session["user_id"], session["user_role"] = "u1", "operator" + return await render_template("_anomaly_card.html", anomalies=anomalies, + back="/hosts/h1", base_resource=base_resource) + + +async def test_a_resources_card_is_one_line_counting_what_waits_with_a_way_to_review(app): + ended = NOW - timedelta(minutes=5) + page = await _card(app, [_view(ended_at=ended), _view(id="a2", metric_name="load_pct", ended_at=ended), + _view(id="a3", review="normal", ended_at=ended)]) + assert "2 to review" in page + assert 'href="/alerts/anomalies?source=host_agent&resource=nas"' in page + assert "This is normal" not in page and "Out of its normal now" not in page # reviewed there + + +async def test_a_card_names_what_is_out_of_its_normal_now(app): + page = await _card(app, [_view(resource_name="nas:/var", metric_name="disk_used_pct")]) + assert "Out of its normal now" in page and "nas:/var" in page and "problem-card" in page + + +async def test_a_card_with_nothing_waiting_or_going_is_not_shown(app): + reviewed = _view(review="normal", ended_at=NOW - timedelta(minutes=5)) + assert "resource-anomalies" not in await _card(app, [reviewed]) + going_but_accepted = _view(review="normal") # the new normal being learned + assert "resource-anomalies" not in await _card(app, [going_but_accepted]) + + async def _widget(app, logged_in, anomalies, to_review): async with app.test_request_context("/status/widget"): if logged_in: diff --git a/tests/core/test_host_page.py b/tests/core/test_host_page.py index 91326b9..972903a 100644 --- a/tests/core/test_host_page.py +++ b/tests/core/test_host_page.py @@ -70,7 +70,7 @@ async def test_the_agent_card_keeps_its_actions_under_manage(app): app, "host_agent", "panel.html", host=host, reg=reg, reporting=True, stale=False, target=target, ansible_available=True, managed_key_set=True) manage = page.index(">Manage agent") - assert page.index("Full metrics →") < manage + assert "Full metrics" not in page # it heads the vitals strip, at the top of the page for action in (">Update agent", ">Rotate token", ">Unregister agent"): assert page.index(action) > manage assert ">Remove" not in page diff --git a/tests/plugins/host_agent/test_host_health.py b/tests/plugins/host_agent/test_host_health.py index 9678436..85fc16c 100644 --- a/tests/plugins/host_agent/test_host_health.py +++ b/tests/plugins/host_agent/test_host_health.py @@ -31,8 +31,8 @@ async def _vitals(app, *, cpu=40.0, mem=50.0, stale=False, seen_min=1): async with app.test_request_context("/plugins/host_agent/vitals/h1"): page = await render_plugin_template( app, "host_agent", "_host_vitals.html", reporting=True, stale=stale, - reg=SimpleNamespace(last_seen_at=datetime.now(timezone.utc) - timedelta(minutes=seen_min), - agent_version="1.9.0"), + reg=SimpleNamespace(host_id="h1", agent_version="1.9.0", + last_seen_at=datetime.now(timezone.utc) - timedelta(minutes=seen_min)), cpu=cpu, mem=mem, disk_root=30.0, load_per_core=20, sparks={"cpu": "", "mem": "", "disk": "", "load": ""}, psi={"cpu": None, "mem": None, "io": None}, worst=worst_reading(readings, TH)) @@ -53,6 +53,12 @@ async def test_a_quiet_host_says_so(app): assert "Every reading is within its thresholds." in await _vitals(app) +async def test_the_strip_at_the_top_of_the_page_leads_to_the_full_metrics(app): + """The temperatures, charts and every mount are on the full-metrics page; + its link sat in the Agent card below the fold and was missed.""" + assert 'href="/plugins/host_agent/h1/"' in await _vitals(app) + + async def test_a_stale_host_says_since_when_before_any_reading(app): page = await _vitals(app, mem=93.0, stale=True, seen_min=20) assert 'Stale' in page