bvandeusenandClaude Opus 5.5
5569f9934e
#5740 "A metric that departs from its normal is recorded as an anomaly to review, and is findable after it ends", milestone 513. - Every minute each learned key's last 15 minutes is judged against its hour's normal: above the usual hold x1.25 and by the metric's floor opens an occurrence, which ends when the level comes back. - metric_anomalies holds one row per item to review; a repeat while it still waits joins it as an occurrence (metric_anomaly_spans). - An ended row nobody reviews within anomalies.review_days (14, Settings -> Thresholds) lapses; an ongoing one never does. - Learning skips every hour an occurrence covers unless its anomaly was marked normal: no normal is learned without review. - Alerts -> Anomalies lists those to review first, then the reviewed and lapsed under day dividers; marking is operator/admin and audited. - The Overview widget and get_overview warn of open ones and count those to review; the host page shows its own; the weekly report lists the week's and the lapsed; MCP list_anomalies and review_anomaly. - Reviewed and lapsed rows are pruned at data.retention_days, and kept in backups like alert history. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
345 lines
15 KiB
Python
345 lines
15 KiB
Python
from __future__ import annotations
|
|
import asyncio
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
from quart import Blueprint, make_response, render_template, request, redirect, url_for, current_app, session
|
|
from sqlalchemy import select
|
|
from steward.ansible import executor, sources as ansible_src
|
|
from steward.alerts import service as alert_svc
|
|
from steward.auth.middleware import require_role
|
|
from steward.core import anomalies as anom, renames
|
|
from steward.core.audit import audit_request, log_audit
|
|
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import UserRole
|
|
from steward.hosts import service as host_svc
|
|
|
|
hosts_bp = Blueprint("hosts", __name__, url_prefix="/hosts")
|
|
|
|
|
|
def _ansible_source_names() -> list[str]:
|
|
try:
|
|
return [s["name"] for s in ansible_src.get_sources(current_app.config.get("ANSIBLE", {}))]
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
@hosts_bp.get("/overview/widget")
|
|
@require_role(UserRole.viewer)
|
|
async def overview_widget():
|
|
"""Dashboard widget: each host's resources, worst first (#5251). Uptime and
|
|
latency belong to the Monitors widget, so this one doesn't read them."""
|
|
async with current_app.db_sessionmaker() as db:
|
|
hosts = (await db.execute(select(Host).order_by(Host.name))).scalars().all()
|
|
host_status = await host_svc.host_status(db, [h.id for h in hosts])
|
|
agent = await host_svc.agent_overview_by_host(
|
|
db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(current_app.config))
|
|
rows = host_svc.host_glance(hosts, host_status, agent, current_app.config.get("THRESHOLDS", {}))
|
|
return await render_template("hosts/overview_widget.html", rows=rows)
|
|
|
|
|
|
@hosts_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def list_hosts():
|
|
# ?add=1 opens the Add host dialog on load (#5291).
|
|
return await _list_page(add={} if request.args.get("add") else None)
|
|
|
|
|
|
async def _list_page(add: dict | None = None):
|
|
"""The Hosts list. `add` opens the Add host dialog: {} for an empty form,
|
|
or {"values": form, "error": msg} when a submit comes back.
|
|
|
|
Search, group and Status filters and a column sort come from the query
|
|
string (#5294). An HTMX request from the filter bar gets the table alone."""
|
|
args = request.args
|
|
sort = args.get("sort") if args.get("sort") in host_svc.HOST_SORTS else "status"
|
|
default_desc = host_svc.HOST_SORTS[sort]
|
|
desc = {"desc": True, "asc": False}.get(args.get("dir"), default_desc)
|
|
f = {"q": args.get("q", "").strip(), "group": args.get("group", ""),
|
|
"state": args.get("state") if args.get("state") in host_svc.HOST_STATES else "",
|
|
"sort": sort, "dir": "desc" if desc else "asc"}
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(select(Host).order_by(Host.name))
|
|
hosts = result.scalars().all()
|
|
ids = [h.id for h in hosts]
|
|
host_status = await host_svc.host_status(db, ids)
|
|
uptime = await host_svc.compute_uptime(db)
|
|
agent = await host_svc.agent_overview_by_host(
|
|
db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(current_app.config))
|
|
groups = await host_svc.host_groups(db, ids)
|
|
group_names = await host_svc.group_names(db)
|
|
|
|
shown = host_svc.filter_hosts(hosts, q=f["q"], group=f["group"], state=f["state"],
|
|
status=host_status, agent=agent, groups=groups)
|
|
shown = host_svc.sort_hosts(shown, sort, desc, status=host_status, agent=agent, uptime=uptime)
|
|
|
|
def filter_url(**change) -> str:
|
|
"""This list's URL with some filters changed; defaults are left out."""
|
|
want = {**f, **change}
|
|
if want["dir"] == ("desc" if host_svc.HOST_SORTS[want["sort"]] else "asc"):
|
|
want["dir"] = ""
|
|
if want["sort"] == "status":
|
|
want["sort"] = ""
|
|
return url_for("hosts.list_hosts", **{k: v for k, v in want.items() if v})
|
|
|
|
def sort_url(key: str) -> str:
|
|
"""A column header's link: its own default direction, or the other
|
|
way when it is already the sort."""
|
|
if key == sort:
|
|
return filter_url(sort=key, dir="asc" if desc else "desc")
|
|
return filter_url(sort=key, dir="desc" if host_svc.HOST_SORTS[key] else "asc")
|
|
|
|
ctx = dict(hosts=shown, total=len(hosts), host_status=host_status, uptime=uptime, agent=agent,
|
|
groups=groups, group_names=group_names, f=f, filter_url=filter_url, sort_url=sort_url,
|
|
states=host_svc.HOST_STATES, no_group=host_svc.NO_GROUP, add=add,
|
|
clock=host_svc.clock_trouble(
|
|
agent, (current_app.config.get("THRESHOLDS", {}).get("clock") or {"warn": 30})["warn"]),
|
|
host_ids={h.name: h.id for h in hosts})
|
|
# The filter bar's swap gets the table alone. A history restore (Back to a
|
|
# pushed URL) is an HTMX request too but wants the page, so key on the
|
|
# target; Vary keeps the browser from caching one as the other.
|
|
partial = request.headers.get("HX-Target") == "hosts-table" \
|
|
and not request.headers.get("HX-History-Restore-Request")
|
|
resp = await make_response(await render_template(
|
|
"hosts/_table.html" if partial else "hosts/list.html", **ctx))
|
|
resp.vary.add("HX-Request")
|
|
return resp
|
|
|
|
|
|
@hosts_bp.get("/<host_id>")
|
|
@require_role(UserRole.viewer)
|
|
async def host_detail(host_id: str):
|
|
"""The host hub: monitors + agent (embedded fragment) + Ansible, one view."""
|
|
from steward.models.ansible_inventory import AnsibleTarget
|
|
from sqlalchemy.orm import selectinload
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
host = (await db.execute(
|
|
select(Host).where(Host.id == host_id))).scalar_one_or_none()
|
|
if host is None:
|
|
return "Not found", 404
|
|
monitors = await host_svc.host_monitors(db, host_id)
|
|
uptime = (await host_svc.compute_uptime(db)).get(host_id)
|
|
monitor_keys = {(getattr(d["monitor"].type, "value", d["monitor"].type), d["monitor"].name)
|
|
for d in monitors}
|
|
alerts = [r for r in await alert_svc.active_rows(db, datetime.now(timezone.utc))
|
|
if alert_svc.watches_host(r["rule"], host.name, monitor_keys)]
|
|
linked_target = (await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
|
|
.options(selectinload(AnsibleTarget.groups)))).scalar_one_or_none()
|
|
linkable_targets = (await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id.is_(None))
|
|
.order_by(AnsibleTarget.name))).scalars().all()
|
|
# The host's metric anomalies, its disks and interfaces included (#5740).
|
|
anomalies = await anom.views(db, await anom.list_anomalies(
|
|
db, source="host_agent", resource=host.name,
|
|
since=datetime.now(timezone.utc) - timedelta(days=30), limit=10), links=False)
|
|
|
|
return await render_template(
|
|
"hosts/detail.html",
|
|
host=host, monitors=monitors, uptime=uptime, alerts=alerts, anomalies=anomalies,
|
|
linked_target=linked_target, linkable_targets=linkable_targets,
|
|
ansible_sources=_ansible_source_names(),
|
|
)
|
|
|
|
|
|
@hosts_bp.post("/")
|
|
@require_role(UserRole.operator)
|
|
async def create_host():
|
|
form = await request.form
|
|
host = Host(
|
|
name=form["name"].strip(),
|
|
address=form["address"].strip(),
|
|
)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
try:
|
|
await host_svc.check_name_free(db, host.name)
|
|
except ValueError as exc:
|
|
return await _list_page(add={"values": form, "error": str(exc)}), 400
|
|
db.add(host)
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"host.created", entity_type="host", entity_id=host.name,
|
|
detail={"address": host.address})
|
|
# Land on the new host's hub so monitors/agent/ansible can be set up next.
|
|
return redirect(url_for("hosts.host_detail", host_id=host.id))
|
|
|
|
|
|
@hosts_bp.get("/<host_id>/edit")
|
|
@require_role(UserRole.operator)
|
|
async def edit_host(host_id: str):
|
|
# Pure host config (name/address/monitors). Agent, Ansible target linking,
|
|
# and playbook runs live on the host detail page (the hub), not here.
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(select(Host).where(Host.id == host_id))
|
|
host = result.scalar_one_or_none()
|
|
if host is None:
|
|
return "Not found", 404
|
|
|
|
return await render_template("hosts/form.html", host=host)
|
|
|
|
|
|
@hosts_bp.post("/<host_id>")
|
|
@require_role(UserRole.operator)
|
|
async def update_host(host_id: str):
|
|
form = await request.form
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(Host).where(Host.id == host_id))
|
|
host = result.scalar_one_or_none()
|
|
if host is None:
|
|
return "Not found", 404
|
|
old_name, name = host.name, form["name"].strip()
|
|
if name != old_name:
|
|
try:
|
|
await host_svc.check_name_free(db, name, exclude_id=host.id)
|
|
except ValueError as exc:
|
|
return await render_template("hosts/form.html", host=host, values=form,
|
|
error=str(exc)), 400
|
|
host.name = name
|
|
host.address = form["address"].strip()
|
|
# Agent and container metrics, and the rules and windows on them,
|
|
# go by the host's name: move them with it.
|
|
moved = await renames.carry_rename(db, renames.HOST_SOURCES, old_name, name)
|
|
if moved:
|
|
await audit_request("host.renamed", "host", name,
|
|
{"host_id": host_id, "renamed_from": old_name, "carried": moved})
|
|
return redirect(url_for("hosts.host_detail", host_id=host_id))
|
|
|
|
|
|
@hosts_bp.post("/<host_id>/ansible-link")
|
|
@require_role(UserRole.operator)
|
|
async def ansible_link(host_id: str):
|
|
"""Link, unlink, or create an AnsibleTarget from this host."""
|
|
from steward.models.ansible_inventory import AnsibleTarget
|
|
|
|
form = await request.form
|
|
action = form.get("action", "")
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
if action == "unlink":
|
|
linked_result = await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
|
|
)
|
|
linked = linked_result.scalar_one_or_none()
|
|
if linked:
|
|
linked.host_id = None
|
|
|
|
elif action == "link":
|
|
target_id = form.get("target_id", "").strip()
|
|
if target_id:
|
|
result = await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.id == target_id)
|
|
)
|
|
target = result.scalar_one_or_none()
|
|
if target:
|
|
# Clear any previous link for this host first
|
|
old_result = await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
|
|
)
|
|
old = old_result.scalar_one_or_none()
|
|
if old and old.id != target_id:
|
|
old.host_id = None
|
|
target.host_id = host_id
|
|
|
|
elif action == "create":
|
|
host_result = await db.execute(select(Host).where(Host.id == host_id))
|
|
host = host_result.scalar_one_or_none()
|
|
if host:
|
|
new_target = AnsibleTarget(
|
|
id=str(uuid.uuid4()),
|
|
name=host.name,
|
|
address=host.address,
|
|
host_id=host_id,
|
|
)
|
|
db.add(new_target)
|
|
|
|
return redirect(f"/hosts/{host_id}")
|
|
|
|
|
|
@hosts_bp.post("/<host_id>/run-playbook")
|
|
@require_role(UserRole.operator)
|
|
async def run_playbook(host_id: str):
|
|
"""Run an Ansible playbook against just this host (ephemeral one-host inventory)."""
|
|
form = await request.form
|
|
async with current_app.db_sessionmaker() as db:
|
|
host = (await db.execute(select(Host).where(Host.id == host_id))).scalar_one_or_none()
|
|
if host is None:
|
|
return "Not found", 404
|
|
|
|
source_name = (form.get("source_name", "") or "").strip()
|
|
playbook = (form.get("playbook_path", "") or "").strip()
|
|
if not source_name or not playbook:
|
|
return "source and playbook are required", 400
|
|
try:
|
|
srcs = ansible_src.get_sources(current_app.config.get("ANSIBLE", {}))
|
|
except Exception:
|
|
return "Ansible sources are misconfigured", 400
|
|
source = next((s for s in srcs if s["name"] == source_name), None)
|
|
if source is None:
|
|
return "Source not found", 404
|
|
if playbook not in ansible_src.discover_playbooks(source["path"]):
|
|
return f"Playbook '{playbook}' not found in source '{source_name}'", 404
|
|
|
|
# Shared parser: discovered var__ fields → extra_vars_map, secret__ → secret_vars
|
|
# (unpersisted), plus tags/check. No --limit — there's exactly one host.
|
|
from steward.ansible.routes import _parse_run_params, _run_vars, check_run_form
|
|
params_or_none, secret_vars, err = _parse_run_params(form)
|
|
if err:
|
|
return err, 400
|
|
err = check_run_form(form, source, playbook, params_or_none, secret_vars)
|
|
if err:
|
|
return err, 400
|
|
|
|
run_id = str(uuid.uuid4())
|
|
inv_content = ansible_src.host_inventory_content(host)
|
|
run = AnsibleRun(
|
|
id=run_id,
|
|
playbook_path=playbook,
|
|
inventory_path=f"host: {host.name}",
|
|
source_name=source_name,
|
|
triggered_by=session.get("user_id"),
|
|
status=AnsibleRunStatus.running,
|
|
params=params_or_none,
|
|
)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
db.add(run)
|
|
|
|
task = asyncio.create_task(
|
|
executor.start_run(
|
|
current_app._get_current_object(), # type: ignore[attr-defined]
|
|
run_id, playbook, f"host: {host.name}", source["path"],
|
|
params_or_none, inv_content, secret_vars=secret_vars,
|
|
)
|
|
)
|
|
task.add_done_callback(
|
|
lambda t: t.exception() and current_app.logger.error(
|
|
"Host playbook run %s raised: %s", run_id, t.exception()))
|
|
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"ansible.host_run", entity_type="host", entity_id=host.name,
|
|
detail={"playbook": playbook, "source": source_name, "run_id": run_id,
|
|
"check": bool((params_or_none or {}).get("check")),
|
|
"vars": sorted(_run_vars(params_or_none, secret_vars))})
|
|
return redirect(url_for("ansible.run_detail", run_id=run_id))
|
|
|
|
|
|
@hosts_bp.post("/<host_id>/delete")
|
|
@require_role(UserRole.admin)
|
|
async def delete_host(host_id: str):
|
|
host_name = None
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(Host).where(Host.id == host_id))
|
|
host = result.scalar_one_or_none()
|
|
if host:
|
|
host_name = host.name
|
|
await db.delete(host)
|
|
if host_name:
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"host.deleted", entity_type="host", entity_id=host_name)
|
|
return redirect(url_for("hosts.list_hosts"))
|