diff --git a/plugins/prometheus/mcp_tools.py b/plugins/prometheus/mcp_tools.py index 21fbba2..38fef8a 100644 --- a/plugins/prometheus/mcp_tools.py +++ b/plugins/prometheus/mcp_tools.py @@ -19,7 +19,7 @@ from .models import PrometheusTarget from .routes import group_series from .service import OWN_METRICS, SOURCE, target_state -REACHES = ("down", "stale", "pending", "up", "off") +REACHES = ("down", "stale", "pending", "up", "paused") _ORDER = {r: i for i, r in enumerate(REACHES)} @@ -37,7 +37,7 @@ async def list_prometheus_targets() -> dict: """The /metrics endpoints Steward scrapes, not answering first. `reach`: "down" (the last scrape failed; `says` is why), "stale" (not - scraped recently), "pending", "up" or "off". `series_kept` / + scraped recently), "pending", "up" or "paused". `series_kept` / `series_dropped` count the newest scrape's series the keep/drop patterns let through or left out; `series_capped` those past `series_cap`, which were not stored. get_prometheus_target lists a target's series; @@ -94,7 +94,7 @@ async def get_prometheus_target(target: str, search: str = "", limit: int = 100) def target_findings(views: list[dict]) -> list[dict]: - """A target whose last scrape failed. Stale and off are the plugin or + """A target whose last scrape failed. Stale and paused are the plugin or the admin's choice, not the endpoint, so they are left out.""" return [finding("warning", "prometheus_target_down", v["name"], f"Prometheus target {v['name']} isn't answering: {v['says']}", diff --git a/plugins/prometheus/routes.py b/plugins/prometheus/routes.py index dcef850..0d79c9a 100644 --- a/plugins/prometheus/routes.py +++ b/plugins/prometheus/routes.py @@ -25,7 +25,7 @@ from .service import (OWN_METRICS, SOURCE, FormError, fold_states, form_values, prometheus_bp = Blueprint("prometheus", __name__, template_folder="templates") -_REACH_ORDER = {"down": 0, "stale": 1, "pending": 2, "up": 3, "off": 4} +_REACH_ORDER = {"down": 0, "stale": 1, "pending": 2, "up": 3, "paused": 4} def _actor() -> tuple[str | None, str]: @@ -40,7 +40,8 @@ def view(target: PrometheusTarget, own: dict, now: datetime) -> dict: """What a page says about a target: its state in words and the numbers from its newest scrape.""" state = target_state(target, now) - return {"id": target.id, "name": target.name, "url": target.url, "reach": state.reach, + return {"id": target.id, "name": target.name, "url": target.url, "enabled": target.enabled, + "reach": state.reach, "says": state.says, "scraped_at": target.last_scraped_at, "ok_at": target.last_ok_at, "interval": target.interval_seconds, "kept": target.last_kept, "dropped": target.last_dropped, "capped": target.last_capped, "cap": target.series_cap, @@ -79,8 +80,7 @@ def _typed(form, target: PrometheusTarget | None = None) -> dict: return {"name": form.get("name", ""), "url": form.get("url", ""), "interval_seconds": form.get("interval_seconds", ""), "series_cap": form.get("series_cap", ""), "keep": form.get("keep", ""), "drop": form.get("drop", ""), - "verify_tls": bool(form.get("verify_tls")), "enabled": bool(form.get("enabled")), - "has_token": bool(target and target.token)} + "verify_tls": bool(form.get("verify_tls")), "has_token": bool(target and target.token)} async def _name_taken(db, name: str, other_than: str | None = None) -> bool: @@ -176,6 +176,27 @@ async def save_target(target_id: str): return redirect(f"/plugins/prometheus/targets/{target_id}") +@prometheus_bp.post("/targets//toggle") +@require_role(UserRole.admin) +async def toggle_target(target_id: str): + """Pause a target, or resume a paused one, from its page or the list.""" + async with current_app.db_sessionmaker() as db: + async with db.begin(): + target = await _target_or_404(db, target_id) + target.enabled = not target.enabled + if target.enabled: + # Resumed: it reads "not scraped yet" until the next scrape, + # rather than stale from before the pause. + target.last_scraped_at = target.last_error = None + enabled, name = target.enabled, target.name + # A rate needs two readings in a row; one from before the pause isn't one. + from . import scheduler + scheduler.RATES.forget(target_id) + await log_audit(current_app, *_actor(), "prometheus.target_resumed" if enabled else "prometheus.target_paused", + entity_type="prometheus_target", entity_id=target_id, detail={"name": name}) + return redirect(request.referrer or f"/plugins/prometheus/targets/{target_id}") + + @prometheus_bp.post("/targets//delete") @require_role(UserRole.admin) async def delete_target(target_id: str): diff --git a/plugins/prometheus/service.py b/plugins/prometheus/service.py index 5ffd774..7f39569 100644 --- a/plugins/prometheus/service.py +++ b/plugins/prometheus/service.py @@ -197,11 +197,10 @@ def form_values(target: PrometheusTarget | None) -> dict: """The form's fields for an existing target. The token is never sent back to the browser; a blank token field on save keeps the stored one.""" if target is None: - return {"interval_seconds": 60, "series_cap": 1000, "verify_tls": True, "enabled": True, - "keep": "", "drop": ""} + return {"interval_seconds": 60, "series_cap": 1000, "verify_tls": True, "keep": "", "drop": ""} return {"name": target.name, "url": target.url, "interval_seconds": target.interval_seconds, "series_cap": target.series_cap, "verify_tls": target.verify_tls, - "enabled": target.enabled, "keep": "\n".join(target.keep or []), + "keep": "\n".join(target.keep or []), "drop": "\n".join(target.drop or []), "has_token": bool(target.token)} @@ -217,7 +216,8 @@ def _bounded(form, key: str, default: int, low: int, high: int, says: str) -> in def parse_form(form, existing: PrometheusTarget | None = None) -> dict: """The submitted form as the columns to store, the token encrypted. - Raises FormError saying what to fix.""" + Whether it is paused isn't among them: Pause/Resume sets that, and a new + target starts scraping. Raises FormError saying what to fix.""" name = (form.get("name") or "").strip() url = (form.get("url") or "").strip() if not name: @@ -241,7 +241,7 @@ def parse_form(form, existing: PrometheusTarget | None = None) -> dict: return {"name": name, "url": url, "token": token, "verify_tls": bool(form.get("verify_tls")), "interval_seconds": interval, "keep": patterns(form.get("keep") or ""), "drop": patterns(form.get("drop") or ""), - "series_cap": cap, "enabled": bool(form.get("enabled"))} + "series_cap": cap} # ── What the newest scrape says ────────────────────────────────────────────── @@ -253,13 +253,13 @@ STALE_SCRAPES = 3 @dataclass class TargetState: - reach: str # up | down | stale | pending | off + reach: str # up | down | stale | pending | paused says: str # one line, in words def target_state(target: PrometheusTarget, now: datetime) -> TargetState: if not target.enabled: - return TargetState("off", "Not scraped: turned off.") + return TargetState("paused", "Paused: not scraped until it's resumed.") if target.last_scraped_at is None: return TargetState("pending", "Not scraped yet.") if (now - target.last_scraped_at).total_seconds() > STALE_SCRAPES * max(target.interval_seconds, 1): diff --git a/plugins/prometheus/templates/prometheus/form.html b/plugins/prometheus/templates/prometheus/form.html index 68b21e4..586c32a 100644 --- a/plugins/prometheus/templates/prometheus/form.html +++ b/plugins/prometheus/templates/prometheus/form.html @@ -51,9 +51,6 @@
- diff --git a/plugins/prometheus/templates/prometheus/index.html b/plugins/prometheus/templates/prometheus/index.html index 90b6e0d..4b020fa 100644 --- a/plugins/prometheus/templates/prometheus/index.html +++ b/plugins/prometheus/templates/prometheus/index.html @@ -23,13 +23,16 @@ {% if t.reach == 'up' %}{{ status_badge('up', 'Answering') }} {% elif t.reach == 'down' %}{{ status_badge('down', 'Not answering') }} {% elif t.reach == 'stale' %}{{ status_badge('stale', 'Not scraped') }} - {% elif t.reach == 'off' %}{{ status_badge('stale', 'Turned off') }} + {% elif t.reach == 'paused' %}{{ status_badge('paused') }} {% else %}{{ status_badge('pending', 'Not scraped yet') }}{% endif %}

{{ t.name }}

{{ t.url }} Series - {% if admin %}Edit{% endif %} + {% if admin %}Edit +
{{ csrf_field() }} + +
{% endif %}
@@ -38,7 +41,7 @@

{%- if t.ok_at %}Last answered {{ t.ok_at|ago }}.{% else %}It has never answered.{% endif %} Tried {{ t.scraped_at|ago }}.

- {% elif t.reach in ('stale', 'off') %} + {% elif t.reach in ('stale', 'paused') %}

{{ t.says }}{% if t.scraped_at %} Last scraped {{ t.scraped_at|ago }}.{% endif %}

{% elif t.reach == 'pending' %}

Not scraped yet: the first scrape runs within seconds of saving.

diff --git a/plugins/prometheus/templates/prometheus/target.html b/plugins/prometheus/templates/prometheus/target.html index b3a874b..8584bfb 100644 --- a/plugins/prometheus/templates/prometheus/target.html +++ b/plugins/prometheus/templates/prometheus/target.html @@ -12,7 +12,10 @@ {% block content %} {% set admin = session.user_role == 'admin' %} {% call page_header(target.name, target.url) %} - {% if admin %}Edit{% endif %} + {% if admin %}Edit +
{{ csrf_field() }} + +
{% endif %} {% endcall %}

@@ -20,7 +23,7 @@ {% elif target.reach == 'down' %}{{ status_badge('down', 'Not answering') }} {{ target.says }} {% if target.ok_at %}Last answered {{ target.ok_at|ago }}; the values below are from then.{% else %}It has never answered.{% endif %} {% elif target.reach == 'stale' %}{{ status_badge('stale', 'Not scraped') }} {{ target.says }} - {% elif target.reach == 'off' %}{{ status_badge('stale', 'Turned off') }} {{ target.says }} + {% elif target.reach == 'paused' %}{{ status_badge('paused') }} {{ target.says }} {% else %}{{ status_badge('pending', 'Not scraped yet') }} The first scrape runs within seconds of saving.{% endif %}

{% if target.capped %} diff --git a/plugins/syslog/patterns.py b/plugins/syslog/patterns.py index 3509818..5a7ec56 100644 --- a/plugins/syslog/patterns.py +++ b/plugins/syslog/patterns.py @@ -108,8 +108,7 @@ def parse_form(form) -> dict: if window not in WINDOWS: window = 15 return {"name": name, "phrases": phrases, "max_severity": max_severity, - "source_id": form.get("source_id") or None, "window_minutes": window, - "enabled": "enabled" in form} + "source_id": form.get("source_id") or None, "window_minutes": window} def matches(pattern, severity: int, message: str) -> bool: diff --git a/plugins/syslog/routes.py b/plugins/syslog/routes.py index 032095e..51ac5bd 100644 --- a/plugins/syslog/routes.py +++ b/plugins/syslog/routes.py @@ -381,13 +381,13 @@ async def add_preset(key: str): @syslog_bp.get("/patterns/new") @require_role(UserRole.admin) async def new_pattern(): - return await _render_pattern_form(None, {"window_minutes": 15, "enabled": True}) + return await _render_pattern_form(None, {"window_minutes": 15}) def _typed_pattern(form) -> dict: return {"name": form.get("name", ""), "phrases": form.get("phrases", ""), "max_severity": form.get("max_severity", ""), "source_id": form.get("source_id", ""), - "window_minutes": form.get("window_minutes", 15, type=int), "enabled": "enabled" in form} + "window_minutes": form.get("window_minutes", 15, type=int)} @syslog_bp.post("/patterns/new") @@ -426,7 +426,7 @@ async def edit_pattern(pattern_id: str): return await _render_pattern_form(p, { "name": p.name, "phrases": "\n".join(p.phrases or []), "max_severity": "" if p.max_severity is None else str(p.max_severity), - "source_id": p.source_id or "", "window_minutes": p.window_minutes, "enabled": p.enabled}) + "source_id": p.source_id or "", "window_minutes": p.window_minutes}) @syslog_bp.post("/patterns//edit") @@ -451,6 +451,20 @@ async def save_pattern(pattern_id: str): return redirect(PATTERNS) +@syslog_bp.post("/patterns//toggle") +@require_role(UserRole.admin) +async def toggle_pattern(pattern_id: str): + """Pause a pattern's counting, or resume it, from the patterns list.""" + async with current_app.db_sessionmaker() as db: + async with db.begin(): + p = await _pattern_or_404(db, pattern_id) + p.enabled = not p.enabled + enabled, name = p.enabled, p.name + await log_audit(current_app, *_actor(), "syslog.pattern_resumed" if enabled else "syslog.pattern_paused", + entity_type="syslog_pattern", entity_id=pattern_id, detail={"name": name}) + return redirect(PATTERNS) + + @syslog_bp.post("/patterns//delete") @require_role(UserRole.admin) async def delete_pattern(pattern_id: str): diff --git a/plugins/syslog/templates/syslog/pattern_form.html b/plugins/syslog/templates/syslog/pattern_form.html index 938985d..44a8c0e 100644 --- a/plugins/syslog/templates/syslog/pattern_form.html +++ b/plugins/syslog/templates/syslog/pattern_form.html @@ -45,12 +45,6 @@ The count is of matches this far back, so an alert on it clears this long after the last one. -
- -
diff --git a/plugins/syslog/templates/syslog/patterns.html b/plugins/syslog/templates/syslog/patterns.html index f61c6f3..b7e9a8b 100644 --- a/plugins/syslog/templates/syslog/patterns.html +++ b/plugins/syslog/templates/syslog/patterns.html @@ -1,7 +1,7 @@ {# plugins/syslog/templates/syslog/patterns.html — telling phrases counted in each source's lines every minute, so an alert rule can watch them (#6001). #} {% extends "base.html" %} -{% from "_macros.html" import crumbs, page_header, empty_state %} +{% from "_macros.html" import crumbs, page_header, empty_state, status_badge %} {% block title %}Patterns — Syslog — Steward{% endblock %} {% block breadcrumb %}{{ crumbs([("Syslog", "/plugins/syslog/"), ("Patterns", "")]) }}{% endblock %} {% block content %} @@ -23,7 +23,7 @@ {% for p in patterns %} - {{ p.name }}{% if not p.enabled %} (off){% endif %}
{{ p.metric }}
+ {{ p.name }}{% if not p.enabled %} {{ status_badge('paused') }}{% endif %}
{{ p.metric }}
{% if p.phrases %}{{ p.phrases[:3]|join(" · ") }}{% if p.phrases|length > 3 %} · +{{ p.phrases|length - 3 }} more{% endif %}{% endif %} {% if p.severity %}
{{ p.severity }} and worse
{% endif %} @@ -37,7 +37,10 @@ {% if can_alert %} {% if p.alert_resource %}Alert on it{% endif %} - {% if admin %}Edit{% endif %} + {% if admin %}Edit +
{{ csrf_field() }} + +
{% endif %} {% endif %} diff --git a/tests/integration/test_syslog.py b/tests/integration/test_syslog.py index 5c58a4b..ebc723e 100644 --- a/tests/integration/test_syslog.py +++ b/tests/integration/test_syslog.py @@ -372,12 +372,13 @@ def test_patterns_page_adds_edits_and_removes_and_a_rename_moves_counts(app): assert "at least one phrase" in await refused.get_data(as_text=True) made = await admin.post("/plugins/syslog/patterns/new", headers=csrf, form={"name": "Quota full", "phrases": "100%\nquota", "max_severity": "4", - "window_minutes": "60", "enabled": "on"}) + "window_minutes": "60"}) assert made.status_code == 302 async with app.db_sessionmaker() as s: rows = {p.key: p for p in (await s.execute(select(models.SyslogPattern))).scalars()} assert set(rows) == {"nfs", "quota_full"} assert (rows["quota_full"].phrases, rows["quota_full"].max_severity) == (["100%", "quota"], 4) + assert rows["quota_full"].enabled # a new pattern counts assert rows["nfs"].phrases[0] == "nfsd:" await patterns.record_counts(app, now) @@ -391,13 +392,24 @@ def test_patterns_page_adds_edits_and_removes_and_a_rename_moves_counts(app): )).get_data(as_text=True) assert '