fix(prometheus, syslog): a target or pattern pauses with a Pause/Resume button, not an "on" tick box in its form (#6419)
Monitors, schedules and maintenance plans already pause with a Pause/Resume button and say "Paused" where they're listed. Prometheus targets and syslog patterns only had a tick box inside the edit form, so a target refused by its endpoint couldn't be stopped from its page. - Both get a Pause/Resume button (target page and list; patterns list) posting to /toggle, recorded in the audit log, and a Paused badge. - The forms lose the tick box. A new one starts running; an edit keeps whether it is paused. - A target's "off" state is now "paused", in its pages and MCP tools. A resumed target reads "not scraped yet" until its next scrape, and its counter rates start over. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -19,7 +19,7 @@ from .models import PrometheusTarget
|
|||||||
from .routes import group_series
|
from .routes import group_series
|
||||||
from .service import OWN_METRICS, SOURCE, target_state
|
from .service import OWN_METRICS, SOURCE, target_state
|
||||||
|
|
||||||
REACHES = ("down", "stale", "pending", "up", "off")
|
REACHES = ("down", "stale", "pending", "up", "paused")
|
||||||
_ORDER = {r: i for i, r in enumerate(REACHES)}
|
_ORDER = {r: i for i, r in enumerate(REACHES)}
|
||||||
|
|
||||||
|
|
||||||
@@ -37,7 +37,7 @@ async def list_prometheus_targets() -> dict:
|
|||||||
"""The /metrics endpoints Steward scrapes, not answering first.
|
"""The /metrics endpoints Steward scrapes, not answering first.
|
||||||
|
|
||||||
`reach`: "down" (the last scrape failed; `says` is why), "stale" (not
|
`reach`: "down" (the last scrape failed; `says` is why), "stale" (not
|
||||||
scraped recently), "pending", "up" or "off". `series_kept` /
|
scraped recently), "pending", "up" or "paused". `series_kept` /
|
||||||
`series_dropped` count the newest scrape's series the keep/drop patterns
|
`series_dropped` count the newest scrape's series the keep/drop patterns
|
||||||
let through or left out; `series_capped` those past `series_cap`, which
|
let through or left out; `series_capped` those past `series_cap`, which
|
||||||
were not stored. get_prometheus_target lists a target's series;
|
were not stored. get_prometheus_target lists a target's series;
|
||||||
@@ -94,7 +94,7 @@ async def get_prometheus_target(target: str, search: str = "", limit: int = 100)
|
|||||||
|
|
||||||
|
|
||||||
def target_findings(views: list[dict]) -> list[dict]:
|
def target_findings(views: list[dict]) -> list[dict]:
|
||||||
"""A target whose last scrape failed. Stale and off are the plugin or
|
"""A target whose last scrape failed. Stale and paused are the plugin or
|
||||||
the admin's choice, not the endpoint, so they are left out."""
|
the admin's choice, not the endpoint, so they are left out."""
|
||||||
return [finding("warning", "prometheus_target_down", v["name"],
|
return [finding("warning", "prometheus_target_down", v["name"],
|
||||||
f"Prometheus target {v['name']} isn't answering: {v['says']}",
|
f"Prometheus target {v['name']} isn't answering: {v['says']}",
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ from .service import (OWN_METRICS, SOURCE, FormError, fold_states, form_values,
|
|||||||
|
|
||||||
prometheus_bp = Blueprint("prometheus", __name__, template_folder="templates")
|
prometheus_bp = Blueprint("prometheus", __name__, template_folder="templates")
|
||||||
|
|
||||||
_REACH_ORDER = {"down": 0, "stale": 1, "pending": 2, "up": 3, "off": 4}
|
_REACH_ORDER = {"down": 0, "stale": 1, "pending": 2, "up": 3, "paused": 4}
|
||||||
|
|
||||||
|
|
||||||
def _actor() -> tuple[str | None, str]:
|
def _actor() -> tuple[str | None, str]:
|
||||||
@@ -40,7 +40,8 @@ def view(target: PrometheusTarget, own: dict, now: datetime) -> dict:
|
|||||||
"""What a page says about a target: its state in words and the numbers
|
"""What a page says about a target: its state in words and the numbers
|
||||||
from its newest scrape."""
|
from its newest scrape."""
|
||||||
state = target_state(target, now)
|
state = target_state(target, now)
|
||||||
return {"id": target.id, "name": target.name, "url": target.url, "reach": state.reach,
|
return {"id": target.id, "name": target.name, "url": target.url, "enabled": target.enabled,
|
||||||
|
"reach": state.reach,
|
||||||
"says": state.says, "scraped_at": target.last_scraped_at, "ok_at": target.last_ok_at,
|
"says": state.says, "scraped_at": target.last_scraped_at, "ok_at": target.last_ok_at,
|
||||||
"interval": target.interval_seconds, "kept": target.last_kept,
|
"interval": target.interval_seconds, "kept": target.last_kept,
|
||||||
"dropped": target.last_dropped, "capped": target.last_capped, "cap": target.series_cap,
|
"dropped": target.last_dropped, "capped": target.last_capped, "cap": target.series_cap,
|
||||||
@@ -79,8 +80,7 @@ def _typed(form, target: PrometheusTarget | None = None) -> dict:
|
|||||||
return {"name": form.get("name", ""), "url": form.get("url", ""),
|
return {"name": form.get("name", ""), "url": form.get("url", ""),
|
||||||
"interval_seconds": form.get("interval_seconds", ""), "series_cap": form.get("series_cap", ""),
|
"interval_seconds": form.get("interval_seconds", ""), "series_cap": form.get("series_cap", ""),
|
||||||
"keep": form.get("keep", ""), "drop": form.get("drop", ""),
|
"keep": form.get("keep", ""), "drop": form.get("drop", ""),
|
||||||
"verify_tls": bool(form.get("verify_tls")), "enabled": bool(form.get("enabled")),
|
"verify_tls": bool(form.get("verify_tls")), "has_token": bool(target and target.token)}
|
||||||
"has_token": bool(target and target.token)}
|
|
||||||
|
|
||||||
|
|
||||||
async def _name_taken(db, name: str, other_than: str | None = None) -> bool:
|
async def _name_taken(db, name: str, other_than: str | None = None) -> bool:
|
||||||
@@ -176,6 +176,27 @@ async def save_target(target_id: str):
|
|||||||
return redirect(f"/plugins/prometheus/targets/{target_id}")
|
return redirect(f"/plugins/prometheus/targets/{target_id}")
|
||||||
|
|
||||||
|
|
||||||
|
@prometheus_bp.post("/targets/<target_id>/toggle")
|
||||||
|
@require_role(UserRole.admin)
|
||||||
|
async def toggle_target(target_id: str):
|
||||||
|
"""Pause a target, or resume a paused one, from its page or the list."""
|
||||||
|
async with current_app.db_sessionmaker() as db:
|
||||||
|
async with db.begin():
|
||||||
|
target = await _target_or_404(db, target_id)
|
||||||
|
target.enabled = not target.enabled
|
||||||
|
if target.enabled:
|
||||||
|
# Resumed: it reads "not scraped yet" until the next scrape,
|
||||||
|
# rather than stale from before the pause.
|
||||||
|
target.last_scraped_at = target.last_error = None
|
||||||
|
enabled, name = target.enabled, target.name
|
||||||
|
# A rate needs two readings in a row; one from before the pause isn't one.
|
||||||
|
from . import scheduler
|
||||||
|
scheduler.RATES.forget(target_id)
|
||||||
|
await log_audit(current_app, *_actor(), "prometheus.target_resumed" if enabled else "prometheus.target_paused",
|
||||||
|
entity_type="prometheus_target", entity_id=target_id, detail={"name": name})
|
||||||
|
return redirect(request.referrer or f"/plugins/prometheus/targets/{target_id}")
|
||||||
|
|
||||||
|
|
||||||
@prometheus_bp.post("/targets/<target_id>/delete")
|
@prometheus_bp.post("/targets/<target_id>/delete")
|
||||||
@require_role(UserRole.admin)
|
@require_role(UserRole.admin)
|
||||||
async def delete_target(target_id: str):
|
async def delete_target(target_id: str):
|
||||||
|
|||||||
@@ -197,11 +197,10 @@ def form_values(target: PrometheusTarget | None) -> dict:
|
|||||||
"""The form's fields for an existing target. The token is never sent back
|
"""The form's fields for an existing target. The token is never sent back
|
||||||
to the browser; a blank token field on save keeps the stored one."""
|
to the browser; a blank token field on save keeps the stored one."""
|
||||||
if target is None:
|
if target is None:
|
||||||
return {"interval_seconds": 60, "series_cap": 1000, "verify_tls": True, "enabled": True,
|
return {"interval_seconds": 60, "series_cap": 1000, "verify_tls": True, "keep": "", "drop": ""}
|
||||||
"keep": "", "drop": ""}
|
|
||||||
return {"name": target.name, "url": target.url, "interval_seconds": target.interval_seconds,
|
return {"name": target.name, "url": target.url, "interval_seconds": target.interval_seconds,
|
||||||
"series_cap": target.series_cap, "verify_tls": target.verify_tls,
|
"series_cap": target.series_cap, "verify_tls": target.verify_tls,
|
||||||
"enabled": target.enabled, "keep": "\n".join(target.keep or []),
|
"keep": "\n".join(target.keep or []),
|
||||||
"drop": "\n".join(target.drop or []), "has_token": bool(target.token)}
|
"drop": "\n".join(target.drop or []), "has_token": bool(target.token)}
|
||||||
|
|
||||||
|
|
||||||
@@ -217,7 +216,8 @@ def _bounded(form, key: str, default: int, low: int, high: int, says: str) -> in
|
|||||||
|
|
||||||
def parse_form(form, existing: PrometheusTarget | None = None) -> dict:
|
def parse_form(form, existing: PrometheusTarget | None = None) -> dict:
|
||||||
"""The submitted form as the columns to store, the token encrypted.
|
"""The submitted form as the columns to store, the token encrypted.
|
||||||
Raises FormError saying what to fix."""
|
Whether it is paused isn't among them: Pause/Resume sets that, and a new
|
||||||
|
target starts scraping. Raises FormError saying what to fix."""
|
||||||
name = (form.get("name") or "").strip()
|
name = (form.get("name") or "").strip()
|
||||||
url = (form.get("url") or "").strip()
|
url = (form.get("url") or "").strip()
|
||||||
if not name:
|
if not name:
|
||||||
@@ -241,7 +241,7 @@ def parse_form(form, existing: PrometheusTarget | None = None) -> dict:
|
|||||||
return {"name": name, "url": url, "token": token,
|
return {"name": name, "url": url, "token": token,
|
||||||
"verify_tls": bool(form.get("verify_tls")), "interval_seconds": interval,
|
"verify_tls": bool(form.get("verify_tls")), "interval_seconds": interval,
|
||||||
"keep": patterns(form.get("keep") or ""), "drop": patterns(form.get("drop") or ""),
|
"keep": patterns(form.get("keep") or ""), "drop": patterns(form.get("drop") or ""),
|
||||||
"series_cap": cap, "enabled": bool(form.get("enabled"))}
|
"series_cap": cap}
|
||||||
|
|
||||||
|
|
||||||
# ── What the newest scrape says ──────────────────────────────────────────────
|
# ── What the newest scrape says ──────────────────────────────────────────────
|
||||||
@@ -253,13 +253,13 @@ STALE_SCRAPES = 3
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class TargetState:
|
class TargetState:
|
||||||
reach: str # up | down | stale | pending | off
|
reach: str # up | down | stale | pending | paused
|
||||||
says: str # one line, in words
|
says: str # one line, in words
|
||||||
|
|
||||||
|
|
||||||
def target_state(target: PrometheusTarget, now: datetime) -> TargetState:
|
def target_state(target: PrometheusTarget, now: datetime) -> TargetState:
|
||||||
if not target.enabled:
|
if not target.enabled:
|
||||||
return TargetState("off", "Not scraped: turned off.")
|
return TargetState("paused", "Paused: not scraped until it's resumed.")
|
||||||
if target.last_scraped_at is None:
|
if target.last_scraped_at is None:
|
||||||
return TargetState("pending", "Not scraped yet.")
|
return TargetState("pending", "Not scraped yet.")
|
||||||
if (now - target.last_scraped_at).total_seconds() > STALE_SCRAPES * max(target.interval_seconds, 1):
|
if (now - target.last_scraped_at).total_seconds() > STALE_SCRAPES * max(target.interval_seconds, 1):
|
||||||
|
|||||||
@@ -51,9 +51,6 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div style="display:flex;gap:1.5rem;flex-wrap:wrap;margin-bottom:0.9rem;">
|
<div style="display:flex;gap:1.5rem;flex-wrap:wrap;margin-bottom:0.9rem;">
|
||||||
<label style="display:flex;gap:0.5rem;align-items:center;cursor:pointer;">
|
|
||||||
<input type="checkbox" name="enabled" value="on"{% if v.enabled %} checked{% endif %}> Scrape this target
|
|
||||||
</label>
|
|
||||||
<label style="display:flex;gap:0.5rem;align-items:center;cursor:pointer;">
|
<label style="display:flex;gap:0.5rem;align-items:center;cursor:pointer;">
|
||||||
<input type="checkbox" name="verify_tls" value="on"{% if v.verify_tls %} checked{% endif %}> Check its HTTPS certificate
|
<input type="checkbox" name="verify_tls" value="on"{% if v.verify_tls %} checked{% endif %}> Check its HTTPS certificate
|
||||||
</label>
|
</label>
|
||||||
|
|||||||
@@ -23,13 +23,16 @@
|
|||||||
{% if t.reach == 'up' %}{{ status_badge('up', 'Answering') }}
|
{% if t.reach == 'up' %}{{ status_badge('up', 'Answering') }}
|
||||||
{% elif t.reach == 'down' %}{{ status_badge('down', 'Not answering') }}
|
{% elif t.reach == 'down' %}{{ status_badge('down', 'Not answering') }}
|
||||||
{% elif t.reach == 'stale' %}{{ status_badge('stale', 'Not scraped') }}
|
{% elif t.reach == 'stale' %}{{ status_badge('stale', 'Not scraped') }}
|
||||||
{% elif t.reach == 'off' %}{{ status_badge('stale', 'Turned off') }}
|
{% elif t.reach == 'paused' %}{{ status_badge('paused') }}
|
||||||
{% else %}{{ status_badge('pending', 'Not scraped yet') }}{% endif %}
|
{% else %}{{ status_badge('pending', 'Not scraped yet') }}{% endif %}
|
||||||
<h2 id="prom-{{ t.id }}" class="section-title" style="margin:0;"><a href="/plugins/prometheus/targets/{{ t.id }}">{{ t.name }}</a></h2>
|
<h2 id="prom-{{ t.id }}" class="section-title" style="margin:0;"><a href="/plugins/prometheus/targets/{{ t.id }}">{{ t.name }}</a></h2>
|
||||||
<span class="field-hint mono" style="min-width:0;max-width:28rem;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ t.url }}">{{ t.url }}</span>
|
<span class="field-hint mono" style="min-width:0;max-width:28rem;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ t.url }}">{{ t.url }}</span>
|
||||||
<span class="page-actions" style="margin-left:auto;">
|
<span class="page-actions" style="margin-left:auto;">
|
||||||
<a href="/plugins/prometheus/targets/{{ t.id }}" class="btn btn-sm btn-ghost">Series</a>
|
<a href="/plugins/prometheus/targets/{{ t.id }}" class="btn btn-sm btn-ghost">Series</a>
|
||||||
{% if admin %}<a href="/plugins/prometheus/targets/{{ t.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>{% endif %}
|
{% if admin %}<a href="/plugins/prometheus/targets/{{ t.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>
|
||||||
|
<form method="post" action="/plugins/prometheus/targets/{{ t.id }}/toggle">{{ csrf_field() }}
|
||||||
|
<button type="submit" class="btn btn-sm btn-ghost">{{ "Pause" if t.enabled else "Resume" }}</button>
|
||||||
|
</form>{% endif %}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -38,7 +41,7 @@
|
|||||||
<p class="field-hint" style="margin:0 0 0.6rem;">
|
<p class="field-hint" style="margin:0 0 0.6rem;">
|
||||||
{%- if t.ok_at %}Last answered {{ t.ok_at|ago }}.{% else %}It has never answered.{% endif %}
|
{%- if t.ok_at %}Last answered {{ t.ok_at|ago }}.{% else %}It has never answered.{% endif %}
|
||||||
Tried {{ t.scraped_at|ago }}.</p>
|
Tried {{ t.scraped_at|ago }}.</p>
|
||||||
{% elif t.reach in ('stale', 'off') %}
|
{% elif t.reach in ('stale', 'paused') %}
|
||||||
<p class="field-hint" style="margin:0 0 0.6rem;">{{ t.says }}{% if t.scraped_at %} Last scraped {{ t.scraped_at|ago }}.{% endif %}</p>
|
<p class="field-hint" style="margin:0 0 0.6rem;">{{ t.says }}{% if t.scraped_at %} Last scraped {{ t.scraped_at|ago }}.{% endif %}</p>
|
||||||
{% elif t.reach == 'pending' %}
|
{% elif t.reach == 'pending' %}
|
||||||
<p class="field-hint" style="margin:0 0 0.6rem;">Not scraped yet: the first scrape runs within seconds of saving.</p>
|
<p class="field-hint" style="margin:0 0 0.6rem;">Not scraped yet: the first scrape runs within seconds of saving.</p>
|
||||||
|
|||||||
@@ -12,7 +12,10 @@
|
|||||||
{% block content %}
|
{% block content %}
|
||||||
{% set admin = session.user_role == 'admin' %}
|
{% set admin = session.user_role == 'admin' %}
|
||||||
{% call page_header(target.name, target.url) %}
|
{% call page_header(target.name, target.url) %}
|
||||||
{% if admin %}<a href="/plugins/prometheus/targets/{{ target.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>{% endif %}
|
{% if admin %}<a href="/plugins/prometheus/targets/{{ target.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>
|
||||||
|
<form method="post" action="/plugins/prometheus/targets/{{ target.id }}/toggle">{{ csrf_field() }}
|
||||||
|
<button type="submit" class="btn btn-sm btn-ghost">{{ "Pause" if target.enabled else "Resume" }}</button>
|
||||||
|
</form>{% endif %}
|
||||||
{% endcall %}
|
{% endcall %}
|
||||||
|
|
||||||
<p class="page-verdict">
|
<p class="page-verdict">
|
||||||
@@ -20,7 +23,7 @@
|
|||||||
{% elif target.reach == 'down' %}{{ status_badge('down', 'Not answering') }} <span class="tone-crit">{{ target.says }}</span>
|
{% elif target.reach == 'down' %}{{ status_badge('down', 'Not answering') }} <span class="tone-crit">{{ target.says }}</span>
|
||||||
{% if target.ok_at %}Last answered {{ target.ok_at|ago }}; the values below are from then.{% else %}It has never answered.{% endif %}
|
{% if target.ok_at %}Last answered {{ target.ok_at|ago }}; the values below are from then.{% else %}It has never answered.{% endif %}
|
||||||
{% elif target.reach == 'stale' %}{{ status_badge('stale', 'Not scraped') }} {{ target.says }}
|
{% elif target.reach == 'stale' %}{{ status_badge('stale', 'Not scraped') }} {{ target.says }}
|
||||||
{% elif target.reach == 'off' %}{{ status_badge('stale', 'Turned off') }} {{ target.says }}
|
{% elif target.reach == 'paused' %}{{ status_badge('paused') }} {{ target.says }}
|
||||||
{% else %}{{ status_badge('pending', 'Not scraped yet') }} The first scrape runs within seconds of saving.{% endif %}
|
{% else %}{{ status_badge('pending', 'Not scraped yet') }} The first scrape runs within seconds of saving.{% endif %}
|
||||||
</p>
|
</p>
|
||||||
{% if target.capped %}
|
{% if target.capped %}
|
||||||
|
|||||||
@@ -108,8 +108,7 @@ def parse_form(form) -> dict:
|
|||||||
if window not in WINDOWS:
|
if window not in WINDOWS:
|
||||||
window = 15
|
window = 15
|
||||||
return {"name": name, "phrases": phrases, "max_severity": max_severity,
|
return {"name": name, "phrases": phrases, "max_severity": max_severity,
|
||||||
"source_id": form.get("source_id") or None, "window_minutes": window,
|
"source_id": form.get("source_id") or None, "window_minutes": window}
|
||||||
"enabled": "enabled" in form}
|
|
||||||
|
|
||||||
|
|
||||||
def matches(pattern, severity: int, message: str) -> bool:
|
def matches(pattern, severity: int, message: str) -> bool:
|
||||||
|
|||||||
@@ -381,13 +381,13 @@ async def add_preset(key: str):
|
|||||||
@syslog_bp.get("/patterns/new")
|
@syslog_bp.get("/patterns/new")
|
||||||
@require_role(UserRole.admin)
|
@require_role(UserRole.admin)
|
||||||
async def new_pattern():
|
async def new_pattern():
|
||||||
return await _render_pattern_form(None, {"window_minutes": 15, "enabled": True})
|
return await _render_pattern_form(None, {"window_minutes": 15})
|
||||||
|
|
||||||
|
|
||||||
def _typed_pattern(form) -> dict:
|
def _typed_pattern(form) -> dict:
|
||||||
return {"name": form.get("name", ""), "phrases": form.get("phrases", ""),
|
return {"name": form.get("name", ""), "phrases": form.get("phrases", ""),
|
||||||
"max_severity": form.get("max_severity", ""), "source_id": form.get("source_id", ""),
|
"max_severity": form.get("max_severity", ""), "source_id": form.get("source_id", ""),
|
||||||
"window_minutes": form.get("window_minutes", 15, type=int), "enabled": "enabled" in form}
|
"window_minutes": form.get("window_minutes", 15, type=int)}
|
||||||
|
|
||||||
|
|
||||||
@syslog_bp.post("/patterns/new")
|
@syslog_bp.post("/patterns/new")
|
||||||
@@ -426,7 +426,7 @@ async def edit_pattern(pattern_id: str):
|
|||||||
return await _render_pattern_form(p, {
|
return await _render_pattern_form(p, {
|
||||||
"name": p.name, "phrases": "\n".join(p.phrases or []),
|
"name": p.name, "phrases": "\n".join(p.phrases or []),
|
||||||
"max_severity": "" if p.max_severity is None else str(p.max_severity),
|
"max_severity": "" if p.max_severity is None else str(p.max_severity),
|
||||||
"source_id": p.source_id or "", "window_minutes": p.window_minutes, "enabled": p.enabled})
|
"source_id": p.source_id or "", "window_minutes": p.window_minutes})
|
||||||
|
|
||||||
|
|
||||||
@syslog_bp.post("/patterns/<pattern_id>/edit")
|
@syslog_bp.post("/patterns/<pattern_id>/edit")
|
||||||
@@ -451,6 +451,20 @@ async def save_pattern(pattern_id: str):
|
|||||||
return redirect(PATTERNS)
|
return redirect(PATTERNS)
|
||||||
|
|
||||||
|
|
||||||
|
@syslog_bp.post("/patterns/<pattern_id>/toggle")
|
||||||
|
@require_role(UserRole.admin)
|
||||||
|
async def toggle_pattern(pattern_id: str):
|
||||||
|
"""Pause a pattern's counting, or resume it, from the patterns list."""
|
||||||
|
async with current_app.db_sessionmaker() as db:
|
||||||
|
async with db.begin():
|
||||||
|
p = await _pattern_or_404(db, pattern_id)
|
||||||
|
p.enabled = not p.enabled
|
||||||
|
enabled, name = p.enabled, p.name
|
||||||
|
await log_audit(current_app, *_actor(), "syslog.pattern_resumed" if enabled else "syslog.pattern_paused",
|
||||||
|
entity_type="syslog_pattern", entity_id=pattern_id, detail={"name": name})
|
||||||
|
return redirect(PATTERNS)
|
||||||
|
|
||||||
|
|
||||||
@syslog_bp.post("/patterns/<pattern_id>/delete")
|
@syslog_bp.post("/patterns/<pattern_id>/delete")
|
||||||
@require_role(UserRole.admin)
|
@require_role(UserRole.admin)
|
||||||
async def delete_pattern(pattern_id: str):
|
async def delete_pattern(pattern_id: str):
|
||||||
|
|||||||
@@ -45,12 +45,6 @@
|
|||||||
</select>
|
</select>
|
||||||
<span class="field-hint">The count is of matches this far back, so an alert on it clears this long after the last one.</span>
|
<span class="field-hint">The count is of matches this far back, so an alert on it clears this long after the last one.</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="field">
|
|
||||||
<label class="field-check">
|
|
||||||
<input type="checkbox" name="enabled"{% if v.enabled %} checked{% endif %}>
|
|
||||||
Counted: off keeps the pattern without counting it
|
|
||||||
</label>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
<div class="page-actions">
|
<div class="page-actions">
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{# plugins/syslog/templates/syslog/patterns.html — telling phrases counted in
|
{# plugins/syslog/templates/syslog/patterns.html — telling phrases counted in
|
||||||
each source's lines every minute, so an alert rule can watch them (#6001). #}
|
each source's lines every minute, so an alert rule can watch them (#6001). #}
|
||||||
{% extends "base.html" %}
|
{% extends "base.html" %}
|
||||||
{% from "_macros.html" import crumbs, page_header, empty_state %}
|
{% from "_macros.html" import crumbs, page_header, empty_state, status_badge %}
|
||||||
{% block title %}Patterns — Syslog — Steward{% endblock %}
|
{% block title %}Patterns — Syslog — Steward{% endblock %}
|
||||||
{% block breadcrumb %}{{ crumbs([("Syslog", "/plugins/syslog/"), ("Patterns", "")]) }}{% endblock %}
|
{% block breadcrumb %}{{ crumbs([("Syslog", "/plugins/syslog/"), ("Patterns", "")]) }}{% endblock %}
|
||||||
{% block content %}
|
{% block content %}
|
||||||
@@ -23,7 +23,7 @@
|
|||||||
<tbody>
|
<tbody>
|
||||||
{% for p in patterns %}
|
{% for p in patterns %}
|
||||||
<tr>
|
<tr>
|
||||||
<td>{{ p.name }}{% if not p.enabled %} <span class="field-hint">(off)</span>{% endif %}<div class="field-hint mono">{{ p.metric }}</div></td>
|
<td>{{ p.name }}{% if not p.enabled %} {{ status_badge('paused') }}{% endif %}<div class="field-hint mono">{{ p.metric }}</div></td>
|
||||||
<td>
|
<td>
|
||||||
{% if p.phrases %}<span class="mono" title="{{ p.phrases|join('\n') }}">{{ p.phrases[:3]|join(" · ") }}{% if p.phrases|length > 3 %} · +{{ p.phrases|length - 3 }} more{% endif %}</span>{% endif %}
|
{% if p.phrases %}<span class="mono" title="{{ p.phrases|join('\n') }}">{{ p.phrases[:3]|join(" · ") }}{% if p.phrases|length > 3 %} · +{{ p.phrases|length - 3 }} more{% endif %}</span>{% endif %}
|
||||||
{% if p.severity %}<div class="field-hint">{{ p.severity }} and worse</div>{% endif %}
|
{% if p.severity %}<div class="field-hint">{{ p.severity }} and worse</div>{% endif %}
|
||||||
@@ -37,7 +37,10 @@
|
|||||||
{% if can_alert %}
|
{% if can_alert %}
|
||||||
<td style="white-space:nowrap;">
|
<td style="white-space:nowrap;">
|
||||||
{% if p.alert_resource %}<a class="btn btn-sm btn-ghost" href="/alerts/rules/new?source_module=syslog&resource_name={{ p.alert_resource|urlencode }}&metric_name={{ p.metric|urlencode }}">Alert on it</a>{% endif %}
|
{% if p.alert_resource %}<a class="btn btn-sm btn-ghost" href="/alerts/rules/new?source_module=syslog&resource_name={{ p.alert_resource|urlencode }}&metric_name={{ p.metric|urlencode }}">Alert on it</a>{% endif %}
|
||||||
{% if admin %}<a class="btn btn-sm btn-ghost" href="/plugins/syslog/patterns/{{ p.id }}/edit">Edit</a>{% endif %}
|
{% if admin %}<a class="btn btn-sm btn-ghost" href="/plugins/syslog/patterns/{{ p.id }}/edit">Edit</a>
|
||||||
|
<form method="post" action="/plugins/syslog/patterns/{{ p.id }}/toggle" style="display:inline;margin:0;">{{ csrf_field() }}
|
||||||
|
<button type="submit" class="btn btn-sm btn-ghost">{{ "Pause" if p.enabled else "Resume" }}</button>
|
||||||
|
</form>{% endif %}
|
||||||
</td>
|
</td>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</tr>
|
</tr>
|
||||||
|
|||||||
@@ -372,12 +372,13 @@ def test_patterns_page_adds_edits_and_removes_and_a_rename_moves_counts(app):
|
|||||||
assert "at least one phrase" in await refused.get_data(as_text=True)
|
assert "at least one phrase" in await refused.get_data(as_text=True)
|
||||||
made = await admin.post("/plugins/syslog/patterns/new", headers=csrf,
|
made = await admin.post("/plugins/syslog/patterns/new", headers=csrf,
|
||||||
form={"name": "Quota full", "phrases": "100%\nquota", "max_severity": "4",
|
form={"name": "Quota full", "phrases": "100%\nquota", "max_severity": "4",
|
||||||
"window_minutes": "60", "enabled": "on"})
|
"window_minutes": "60"})
|
||||||
assert made.status_code == 302
|
assert made.status_code == 302
|
||||||
async with app.db_sessionmaker() as s:
|
async with app.db_sessionmaker() as s:
|
||||||
rows = {p.key: p for p in (await s.execute(select(models.SyslogPattern))).scalars()}
|
rows = {p.key: p for p in (await s.execute(select(models.SyslogPattern))).scalars()}
|
||||||
assert set(rows) == {"nfs", "quota_full"}
|
assert set(rows) == {"nfs", "quota_full"}
|
||||||
assert (rows["quota_full"].phrases, rows["quota_full"].max_severity) == (["100%", "quota"], 4)
|
assert (rows["quota_full"].phrases, rows["quota_full"].max_severity) == (["100%", "quota"], 4)
|
||||||
|
assert rows["quota_full"].enabled # a new pattern counts
|
||||||
assert rows["nfs"].phrases[0] == "nfsd:"
|
assert rows["nfs"].phrases[0] == "nfsd:"
|
||||||
|
|
||||||
await patterns.record_counts(app, now)
|
await patterns.record_counts(app, now)
|
||||||
@@ -391,13 +392,24 @@ def test_patterns_page_adds_edits_and_removes_and_a_rename_moves_counts(app):
|
|||||||
)).get_data(as_text=True)
|
)).get_data(as_text=True)
|
||||||
assert '<option value="syslog" selected>' in rule_form # aimed at the pattern's count
|
assert '<option value="syslog" selected>' in rule_form # aimed at the pattern's count
|
||||||
|
|
||||||
# Edited: the key, and so the metric, stay.
|
# Paused from the list (#6419), not from a tick box in the form.
|
||||||
edited = await admin.post(f"/plugins/syslog/patterns/{rows['quota_full'].id}/edit", headers=csrf,
|
quota = rows["quota_full"].id
|
||||||
|
assert (await admin.post(f"/plugins/syslog/patterns/{quota}/toggle", headers=csrf)).status_code == 302
|
||||||
|
page = " ".join((await (await admin.get("/plugins/syslog/patterns")).get_data(as_text=True)).split())
|
||||||
|
assert '<span class="badge badge-dim">Paused</span>' in page and ">Resume</button>" in page
|
||||||
|
form_page = await (await admin.get(f"/plugins/syslog/patterns/{quota}/edit")).get_data(as_text=True)
|
||||||
|
assert 'name="enabled"' not in form_page
|
||||||
|
|
||||||
|
# Edited: the key, and so the metric, stay, and so does the pause.
|
||||||
|
edited = await admin.post(f"/plugins/syslog/patterns/{quota}/edit", headers=csrf,
|
||||||
form={"name": "Quota", "phrases": "quota", "window_minutes": "15"})
|
form={"name": "Quota", "phrases": "quota", "window_minutes": "15"})
|
||||||
assert edited.status_code == 302
|
assert edited.status_code == 302
|
||||||
async with app.db_sessionmaker() as s:
|
async with app.db_sessionmaker() as s:
|
||||||
p = await s.get(models.SyslogPattern, rows["quota_full"].id)
|
p = await s.get(models.SyslogPattern, quota)
|
||||||
assert (p.key, p.name, p.enabled, p.max_severity) == ("quota_full", "Quota", False, None)
|
assert (p.key, p.name, p.enabled, p.max_severity) == ("quota_full", "Quota", False, None)
|
||||||
|
assert (await admin.post(f"/plugins/syslog/patterns/{quota}/toggle", headers=csrf)).status_code == 302
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
assert (await s.get(models.SyslogPattern, quota)).enabled
|
||||||
|
|
||||||
# A renamed source takes its counts with it.
|
# A renamed source takes its counts with it.
|
||||||
renamed = await admin.post(f"/plugins/syslog/sources/{src_id}/edit", headers=csrf,
|
renamed = await admin.post(f"/plugins/syslog/sources/{src_id}/edit", headers=csrf,
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ def _target(name, *, enabled=True, scraped=timedelta(seconds=30), error=None,
|
|||||||
def test_only_a_target_whose_scrape_failed_is_an_overview_finding():
|
def test_only_a_target_whose_scrape_failed_is_an_overview_finding():
|
||||||
views = [t._target_view(x, {}, NOW) | {"_ok_at": x.last_ok_at} for x in (
|
views = [t._target_view(x, {}, NOW) | {"_ok_at": x.last_ok_at} for x in (
|
||||||
_target("archives", error="connection refused"), _target("fine"),
|
_target("archives", error="connection refused"), _target("fine"),
|
||||||
_target("off", enabled=False, error="connection refused"),
|
_target("paused", enabled=False, error="connection refused"),
|
||||||
_target("stale", scraped=timedelta(days=1), error="connection refused"),
|
_target("stale", scraped=timedelta(days=1), error="connection refused"),
|
||||||
_target("new", scraped=None))]
|
_target("new", scraped=None))]
|
||||||
[f] = t.target_findings(views)
|
[f] = t.target_findings(views)
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ from tests.test_style_tokens import hex_colours
|
|||||||
def _view(name, reach, says="Answering.", ok_minutes=None, capped=0):
|
def _view(name, reach, says="Answering.", ok_minutes=None, capped=0):
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
return {"id": f"id-{name}", "name": name, "url": f"http://{name}:8081/metrics", "reach": reach,
|
return {"id": f"id-{name}", "name": name, "url": f"http://{name}:8081/metrics", "reach": reach,
|
||||||
|
"enabled": reach != "paused",
|
||||||
"says": says, "scraped_at": now - timedelta(seconds=20),
|
"says": says, "scraped_at": now - timedelta(seconds=20),
|
||||||
"ok_at": now - timedelta(minutes=ok_minutes) if ok_minutes is not None else None,
|
"ok_at": now - timedelta(minutes=ok_minutes) if ok_minutes is not None else None,
|
||||||
"interval": 60, "kept": 120, "dropped": 30, "capped": capped, "cap": 1000,
|
"interval": 60, "kept": 120, "dropped": 30, "capped": capped, "cap": 1000,
|
||||||
@@ -20,7 +21,8 @@ def _view(name, reach, says="Answering.", ok_minutes=None, capped=0):
|
|||||||
async def test_a_target_not_answering_says_why_and_the_cap_is_shown(app):
|
async def test_a_target_not_answering_says_why_and_the_cap_is_shown(app):
|
||||||
from quart import session
|
from quart import session
|
||||||
targets = [_view("archives", "down", "Answered 401: the token was refused.", ok_minutes=180),
|
targets = [_view("archives", "down", "Answered 401: the token was refused.", ok_minutes=180),
|
||||||
_view("busy", "up", capped=37), _view("new", "pending")]
|
_view("busy", "up", capped=37), _view("new", "pending"),
|
||||||
|
_view("resting", "paused", "Paused: not scraped until it's resumed.")]
|
||||||
async with app.test_request_context("/plugins/prometheus/"):
|
async with app.test_request_context("/plugins/prometheus/"):
|
||||||
session["user_role"] = "admin"
|
session["user_role"] = "admin"
|
||||||
page = await render_plugin_template(app, "prometheus", "prometheus/index.html", targets=targets)
|
page = await render_plugin_template(app, "prometheus", "prometheus/index.html", targets=targets)
|
||||||
@@ -30,6 +32,10 @@ async def test_a_target_not_answering_says_why_and_the_cap_is_shown(app):
|
|||||||
assert "37 series weren't stored" in page and "the cap of 1000" in page
|
assert "37 series weren't stored" in page and "the cap of 1000" in page
|
||||||
assert '<span class="stat-tile-value">42 ms</span>' in page
|
assert '<span class="stat-tile-value">42 ms</span>' in page
|
||||||
assert "the first scrape runs within seconds of saving" in page
|
assert "the first scrape runs within seconds of saving" in page
|
||||||
|
# Paused and resumed from the list (#6419), each row saying which it is.
|
||||||
|
assert '<span class="badge badge-dim">Paused</span>' in page and "not scraped until it's resumed" in page
|
||||||
|
assert 'action="/plugins/prometheus/targets/id-resting/toggle"' in page
|
||||||
|
assert page.count(">Pause</button>") == 3 and page.count(">Resume</button>") == 1
|
||||||
assert 'href="/plugins/prometheus/targets/new"' in page and 'href="/plugins/prometheus/targets/id-busy/edit"' in page
|
assert 'href="/plugins/prometheus/targets/new"' in page and 'href="/plugins/prometheus/targets/id-busy/edit"' in page
|
||||||
main = page[page.index("<main"):]
|
main = page[page.index("<main"):]
|
||||||
assert not hex_colours(main), hex_colours(main)
|
assert not hex_colours(main), hex_colours(main)
|
||||||
@@ -43,7 +49,7 @@ async def test_a_viewer_sees_no_editing_and_an_empty_page_offers_to_add(app):
|
|||||||
targets=[_view("archives", "up")])
|
targets=[_view("archives", "up")])
|
||||||
session["user_role"] = "admin"
|
session["user_role"] = "admin"
|
||||||
empty = await render_plugin_template(app, "prometheus", "prometheus/index.html", targets=[])
|
empty = await render_plugin_template(app, "prometheus", "prometheus/index.html", targets=[])
|
||||||
assert "/edit" not in page and "targets/new" not in page
|
assert "/edit" not in page and "targets/new" not in page and "/toggle" not in page
|
||||||
assert "No targets yet." in empty and 'href="/plugins/prometheus/targets/new"' in empty
|
assert "No targets yet." in empty and 'href="/plugins/prometheus/targets/new"' in empty
|
||||||
|
|
||||||
|
|
||||||
@@ -103,3 +109,4 @@ async def test_the_form_never_echoes_the_token_and_offers_test(app):
|
|||||||
assert 'name="token" type="password"' in page and "kept; type to change" in page
|
assert 'name="token" type="password"' in page and "kept; type to change" in page
|
||||||
assert 'name="clear_token"' in page and 'hx-post="/plugins/prometheus/targets/test"' in page
|
assert 'name="clear_token"' in page and 'hx-post="/plugins/prometheus/targets/test"' in page
|
||||||
assert "fabledarchives_*" in page and "Delete target" in page
|
assert "fabledarchives_*" in page and "Delete target" in page
|
||||||
|
assert 'name="enabled"' not in page # paused from the target's page, not here (#6419)
|
||||||
|
|||||||
@@ -128,13 +128,13 @@ class _Form(dict):
|
|||||||
def test_the_form_checks_what_it_stores_and_keeps_a_blank_token():
|
def test_the_form_checks_what_it_stores_and_keeps_a_blank_token():
|
||||||
values = parse_form(_Form(name="Archives", url="http://archives:8081/metrics", token="s3cret",
|
values = parse_form(_Form(name="Archives", url="http://archives:8081/metrics", token="s3cret",
|
||||||
interval_seconds="30", series_cap="500", keep="fabledarchives_*",
|
interval_seconds="30", series_cap="500", keep="fabledarchives_*",
|
||||||
enabled="on", verify_tls="on"))
|
verify_tls="on"))
|
||||||
assert values["token"].startswith("enc:") or values["token"] == "s3cret"
|
assert values["token"].startswith("enc:") or values["token"] == "s3cret"
|
||||||
assert (values["interval_seconds"], values["series_cap"], values["keep"], values["enabled"]) == (
|
assert (values["interval_seconds"], values["series_cap"], values["keep"]) == (30, 500, ["fabledarchives_*"])
|
||||||
30, 500, ["fabledarchives_*"], True)
|
assert "enabled" not in values # Pause/Resume sets it, not the form (#6419)
|
||||||
existing = PrometheusTarget(name="Archives", url="http://a/metrics", token="enc:v1:kept")
|
existing = PrometheusTarget(name="Archives", url="http://a/metrics", token="enc:v1:kept")
|
||||||
again = parse_form(_Form(name="Archives", url="http://a/metrics"), existing)
|
again = parse_form(_Form(name="Archives", url="http://a/metrics"), existing)
|
||||||
assert again["token"] == "enc:v1:kept" and again["enabled"] is False
|
assert again["token"] == "enc:v1:kept" and "enabled" not in again
|
||||||
cleared = parse_form(_Form(name="Archives", url="http://a/metrics", clear_token="on"), existing)
|
cleared = parse_form(_Form(name="Archives", url="http://a/metrics", clear_token="on"), existing)
|
||||||
assert cleared["token"] == ""
|
assert cleared["token"] == ""
|
||||||
for bad in (dict(name="", url="http://a"), dict(name="a:b", url="http://a"),
|
for bad in (dict(name="", url="http://a"), dict(name="a:b", url="http://a"),
|
||||||
@@ -158,7 +158,7 @@ def test_state_and_due():
|
|||||||
t.last_scraped_at = now - timedelta(minutes=10)
|
t.last_scraped_at = now - timedelta(minutes=10)
|
||||||
assert target_state(t, now).reach == "stale"
|
assert target_state(t, now).reach == "stale"
|
||||||
t.enabled = False
|
t.enabled = False
|
||||||
assert target_state(t, now).reach == "off" and not is_due(t, now)
|
assert target_state(t, now).reach == "paused" and not is_due(t, now)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
@@ -11,12 +11,12 @@ from plugins.syslog.patterns import PRESETS, PatternError, Tally, key_for, match
|
|||||||
|
|
||||||
def test_a_form_reads_into_a_pattern():
|
def test_a_form_reads_into_a_pattern():
|
||||||
v = parse_form(MultiDict({"name": " NFS ", "phrases": "nfsd:\n\n lockd: \n", "max_severity": "3",
|
v = parse_form(MultiDict({"name": " NFS ", "phrases": "nfsd:\n\n lockd: \n", "max_severity": "3",
|
||||||
"source_id": "", "window_minutes": "60", "enabled": "on"}))
|
"source_id": "", "window_minutes": "60"}))
|
||||||
assert v == {"name": "NFS", "phrases": ["nfsd:", "lockd:"], "max_severity": 3, "source_id": None,
|
assert v == {"name": "NFS", "phrases": ["nfsd:", "lockd:"], "max_severity": 3, "source_id": None,
|
||||||
"window_minutes": 60, "enabled": True}
|
"window_minutes": 60} # paused or not is the toggle's, not the form's (#6419)
|
||||||
# A window or severity not offered falls back rather than failing.
|
# A window or severity not offered falls back rather than failing.
|
||||||
v = parse_form(MultiDict({"name": "x", "phrases": "a", "max_severity": "9", "window_minutes": "7"}))
|
v = parse_form(MultiDict({"name": "x", "phrases": "a", "max_severity": "9", "window_minutes": "7"}))
|
||||||
assert (v["max_severity"], v["window_minutes"], v["enabled"]) == (None, 15, False)
|
assert (v["max_severity"], v["window_minutes"]) == (None, 15)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("form, says", [
|
@pytest.mark.parametrize("form, says", [
|
||||||
|
|||||||
Reference in New Issue
Block a user