Files
FabledSteward/steward/hosts/routes.py
T
bvandeusen 988c13d51f
CI / lint (push) Successful in 3s
CI / unit (push) Successful in 7s
CI / integration (push) Successful in 2m25s
CI / publish (push) Successful in 56s
feat(dashboard): phase C richer panels — host time-series graph + cpu sparklines
Milestone 72 phase C — bring the host-view graphs onto the dashboard:
- host_resource_history widget reworked into a real host-view chart: epoch-ms
  linear axis (no Chart.js date adapter), themed like the host-detail charts,
  maintainAspectRatio:false so it fills the resized panel, unique canvas per
  widget instance (wid), and empty states ("pick a host" / "no metrics yet").
  Was previously unusable — it had a broken time axis and no way to choose a host.
- Add a "host" param type: the edit form renders a live dropdown of hosts
  (dashboard routes now pass the host list to the editor); the chosen host_id is
  stored in config and fed to the widget.
- Hosts-overview widget gains a per-row CPU sparkline (last hour) via the shared
  sparkline_svg helper — the host-view at-a-glance trend, on the main widget.

Charts/sparklines render only in the browser, so CI can't exercise them — needs
an operator visual check.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-17 15:18:50 -04:00

529 lines
21 KiB
Python

from __future__ import annotations
import asyncio
import uuid
from datetime import datetime, timedelta, timezone
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
from sqlalchemy import and_, case, select, func
from steward.ansible import executor, sources as ansible_src
from steward.auth.middleware import require_role
from steward.core.audit import log_audit
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
from steward.models.hosts import Host, ProbeType
from steward.models.monitors import PingResult, DnsResult, PingStatus
from steward.models.users import UserRole
hosts_bp = Blueprint("hosts", __name__, url_prefix="/hosts")
def _ansible_source_names() -> list[str]:
try:
return [s["name"] for s in ansible_src.get_sources(current_app.config.get("ANSIBLE", {}))]
except Exception:
return []
async def _compute_uptime(db) -> dict[str, dict]:
"""Return per-host uptime % for 24h, 7d, 30d windows.
Returns {host_id: {"24h": float|None, "7d": float|None, "30d": float|None}}.
"""
now = datetime.now(timezone.utc)
cutoff_30d = now - timedelta(days=30)
cutoff_7d = now - timedelta(days=7)
cutoff_24h = now - timedelta(hours=24)
result = await db.execute(
select(
PingResult.host_id,
# 30d window — all rows in query qualify
func.count(PingResult.id).label("total_30d"),
func.sum(case((PingResult.status == PingStatus.up, 1), else_=0)).label("up_30d"),
# 7d sub-window
func.sum(case((PingResult.probed_at >= cutoff_7d, 1), else_=0)).label("total_7d"),
func.sum(case(
(and_(PingResult.probed_at >= cutoff_7d, PingResult.status == PingStatus.up), 1),
else_=0,
)).label("up_7d"),
# 24h sub-window
func.sum(case((PingResult.probed_at >= cutoff_24h, 1), else_=0)).label("total_24h"),
func.sum(case(
(and_(PingResult.probed_at >= cutoff_24h, PingResult.status == PingStatus.up), 1),
else_=0,
)).label("up_24h"),
)
.where(PingResult.probed_at >= cutoff_30d)
.group_by(PingResult.host_id)
)
def _pct(up, total):
return round(float(up) / float(total) * 100, 2) if total else None
stats: dict[str, dict] = {}
for row in result:
stats[row.host_id] = {
"24h": _pct(row.up_24h, row.total_24h),
"7d": _pct(row.up_7d, row.total_7d),
"30d": _pct(row.up_30d, row.total_30d),
}
return stats
async def _agent_metrics_by_host(db, host_names: list[str]) -> dict[str, dict]:
"""Latest agent cpu/mem per host name from the generic PluginMetric table.
Read directly (PluginMetric is a core model) so the Hosts list can show an
at-a-glance agent column without importing the host_agent plugin.
"""
from steward.models.metrics import PluginMetric
if not host_names:
return {}
wanted = ("cpu_pct", "mem_used_pct")
subq = (
select(
PluginMetric.resource_name, PluginMetric.metric_name,
func.max(PluginMetric.recorded_at).label("m"),
)
.where(
PluginMetric.source_module == "host_agent",
PluginMetric.metric_name.in_(wanted),
PluginMetric.resource_name.in_(host_names),
)
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
).subquery()
rows = (await db.execute(
select(PluginMetric).join(
subq,
(PluginMetric.resource_name == subq.c.resource_name)
& (PluginMetric.metric_name == subq.c.metric_name)
& (PluginMetric.recorded_at == subq.c.m),
)
)).scalars().all()
out: dict[str, dict] = {}
for r in rows:
out.setdefault(r.resource_name, {})[r.metric_name] = r.value
return out
async def _agent_overview_by_host(db, host_names: list[str]) -> dict[str, dict]:
"""Per-host agent glance (cpu/mem/root-disk + freshness) from PluginMetric.
Core-safe (no host_agent import). Freshness compares the latest host-level
sample against the plugin's stale window (config), defaulting to 180s.
"""
from steward.models.metrics import PluginMetric
if not host_names:
return {}
now = datetime.now(timezone.utc)
stale_after = int(
(current_app.config.get("PLUGINS", {}).get("host_agent", {}) or {})
.get("stale_after_seconds", 180))
resources = list(host_names) + [h + ":/" for h in host_names]
subq = (
select(
PluginMetric.resource_name, PluginMetric.metric_name,
func.max(PluginMetric.recorded_at).label("m"),
)
.where(
PluginMetric.source_module == "host_agent",
PluginMetric.resource_name.in_(resources),
PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "disk_used_pct")),
)
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
).subquery()
rows = (await db.execute(
select(PluginMetric).join(
subq,
(PluginMetric.resource_name == subq.c.resource_name)
& (PluginMetric.metric_name == subq.c.metric_name)
& (PluginMetric.recorded_at == subq.c.m),
).where(PluginMetric.source_module == "host_agent")
)).scalars().all()
out: dict[str, dict] = {}
for r in rows:
if r.resource_name.endswith(":/") and r.metric_name == "disk_used_pct":
out.setdefault(r.resource_name[:-2], {})["disk_root"] = r.value
elif r.resource_name in host_names:
d = out.setdefault(r.resource_name, {})
d[r.metric_name] = r.value
if d.get("_ts") is None or r.recorded_at > d["_ts"]:
d["_ts"] = r.recorded_at
for d in out.values():
ts = d.pop("_ts", None)
d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
return out
async def _agent_cpu_sparks_by_host(db, host_names: list[str], hours: int = 1) -> dict[str, str]:
"""{host_name: inline-SVG cpu sparkline} over the last `hours` (core-safe).
A tiny at-a-glance trend in each row — the host-view sparkline, on the widget.
"""
from steward.models.metrics import PluginMetric
from steward.core.status import sparkline_svg
if not host_names:
return {}
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
rows = (await db.execute(
select(PluginMetric.resource_name, PluginMetric.value, PluginMetric.recorded_at)
.where(
PluginMetric.source_module == "host_agent",
PluginMetric.metric_name == "cpu_pct",
PluginMetric.resource_name.in_(host_names),
PluginMetric.recorded_at >= cutoff,
)
.order_by(PluginMetric.recorded_at)
)).all()
by_host: dict[str, list[float]] = {}
for r in rows:
by_host.setdefault(r.resource_name, []).append(r.value)
# Amber line to read as "CPU"; only draw when there are a couple of points.
return {
name: sparkline_svg(vals[-40:], width=64, height=18, stroke="#c8a840")
for name, vals in by_host.items() if len(vals) >= 2
}
@hosts_bp.get("/overview/widget")
@require_role(UserRole.viewer)
async def overview_widget():
"""Dashboard widget: unified per-host monitor + agent glance, linking to the hub."""
async with current_app.db_sessionmaker() as db:
hosts = (await db.execute(select(Host).order_by(Host.name))).scalars().all()
latest_ping_subq = (
select(PingResult.host_id, func.max(PingResult.probed_at).label("max_at"))
.group_by(PingResult.host_id).subquery()
)
latest_pings = {r.host_id: r for r in (await db.execute(
select(PingResult).join(
latest_ping_subq,
(PingResult.host_id == latest_ping_subq.c.host_id)
& (PingResult.probed_at == latest_ping_subq.c.max_at),
))).scalars()}
latest_dns_subq = (
select(DnsResult.host_id, func.max(DnsResult.resolved_at).label("max_at"))
.group_by(DnsResult.host_id).subquery()
)
latest_dns = {r.host_id: r for r in (await db.execute(
select(DnsResult).join(
latest_dns_subq,
(DnsResult.host_id == latest_dns_subq.c.host_id)
& (DnsResult.resolved_at == latest_dns_subq.c.max_at),
))).scalars()}
uptime = await _compute_uptime(db)
host_names = [h.name for h in hosts]
agent = await _agent_overview_by_host(db, host_names)
cpu_sparks = await _agent_cpu_sparks_by_host(db, host_names)
return await render_template(
"hosts/overview_widget.html",
hosts=hosts, latest_pings=latest_pings, latest_dns=latest_dns,
uptime=uptime, agent=agent, cpu_sparks=cpu_sparks,
)
@hosts_bp.get("/")
@require_role(UserRole.viewer)
async def list_hosts():
async with current_app.db_sessionmaker() as db:
result = await db.execute(select(Host).order_by(Host.name))
hosts = result.scalars().all()
# Latest ping result per host
latest_ping_subq = (
select(PingResult.host_id, func.max(PingResult.probed_at).label("max_at"))
.group_by(PingResult.host_id)
.subquery()
)
pr = await db.execute(
select(PingResult).join(
latest_ping_subq,
(PingResult.host_id == latest_ping_subq.c.host_id)
& (PingResult.probed_at == latest_ping_subq.c.max_at),
)
)
latest_pings = {r.host_id: r for r in pr.scalars()}
# Latest DNS result per host
latest_dns_subq = (
select(DnsResult.host_id, func.max(DnsResult.resolved_at).label("max_at"))
.group_by(DnsResult.host_id)
.subquery()
)
dr = await db.execute(
select(DnsResult).join(
latest_dns_subq,
(DnsResult.host_id == latest_dns_subq.c.host_id)
& (DnsResult.resolved_at == latest_dns_subq.c.max_at),
)
)
latest_dns = {r.host_id: r for r in dr.scalars()}
uptime = await _compute_uptime(db)
agent_metrics = await _agent_metrics_by_host(db, [h.name for h in hosts])
return await render_template(
"hosts/list.html",
hosts=hosts,
latest_pings=latest_pings,
latest_dns=latest_dns,
uptime=uptime,
agent_metrics=agent_metrics,
)
@hosts_bp.get("/<host_id>")
@require_role(UserRole.viewer)
async def host_detail(host_id: str):
"""The host hub: monitors + agent (embedded fragment) + Ansible, one view."""
from steward.models.ansible_inventory import AnsibleTarget
from sqlalchemy.orm import selectinload
async with current_app.db_sessionmaker() as db:
host = (await db.execute(
select(Host).where(Host.id == host_id))).scalar_one_or_none()
if host is None:
return "Not found", 404
ping = (await db.execute(
select(PingResult).where(PingResult.host_id == host_id)
.order_by(PingResult.probed_at.desc()).limit(1))).scalar_one_or_none()
dns = (await db.execute(
select(DnsResult).where(DnsResult.host_id == host_id)
.order_by(DnsResult.resolved_at.desc()).limit(1))).scalar_one_or_none()
uptime = (await _compute_uptime(db)).get(host_id)
linked_target = (await db.execute(
select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
.options(selectinload(AnsibleTarget.groups)))).scalar_one_or_none()
linkable_targets = (await db.execute(
select(AnsibleTarget).where(AnsibleTarget.host_id.is_(None))
.order_by(AnsibleTarget.name))).scalars().all()
return await render_template(
"hosts/detail.html",
host=host, ping=ping, dns=dns, uptime=uptime,
linked_target=linked_target, linkable_targets=linkable_targets,
ansible_sources=_ansible_source_names(),
)
@hosts_bp.get("/new")
@require_role(UserRole.operator)
async def new_host():
return await render_template("hosts/form.html", host=None, probe_types=list(ProbeType))
@hosts_bp.post("/")
@require_role(UserRole.operator)
async def create_host():
form = await request.form
host = Host(
name=form["name"].strip(),
address=form["address"].strip(),
probe_type=ProbeType(form.get("probe_type", "tcp")),
probe_port=int(form.get("probe_port") or 80),
ping_enabled="ping_enabled" in form,
dns_enabled="dns_enabled" in form,
dns_expected_ip=form.get("dns_expected_ip", "").strip() or None,
# poll_interval_seconds not exposed in UI yet; per-host scheduling not implemented
)
async with current_app.db_sessionmaker() as db:
async with db.begin():
db.add(host)
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"host.created", entity_type="host", entity_id=host.name,
detail={"address": host.address})
return redirect(url_for("hosts.list_hosts"))
@hosts_bp.get("/<host_id>/edit")
@require_role(UserRole.operator)
async def edit_host(host_id: str):
# Pure host config (name/address/monitors). Agent, Ansible target linking,
# and playbook runs live on the host detail page (the hub), not here.
async with current_app.db_sessionmaker() as db:
result = await db.execute(select(Host).where(Host.id == host_id))
host = result.scalar_one_or_none()
if host is None:
return "Not found", 404
return await render_template(
"hosts/form.html", host=host, probe_types=list(ProbeType))
@hosts_bp.post("/<host_id>")
@require_role(UserRole.operator)
async def update_host(host_id: str):
form = await request.form
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(select(Host).where(Host.id == host_id))
host = result.scalar_one_or_none()
if host is None:
return "Not found", 404
host.name = form["name"].strip()
host.address = form["address"].strip()
host.probe_type = ProbeType(form.get("probe_type", "tcp"))
host.probe_port = int(form.get("probe_port") or 80)
host.ping_enabled = "ping_enabled" in form
host.dns_enabled = "dns_enabled" in form
host.dns_expected_ip = form.get("dns_expected_ip", "").strip() or None
# poll_interval_seconds not exposed in UI yet
return redirect(url_for("hosts.list_hosts"))
@hosts_bp.post("/<host_id>/ansible-link")
@require_role(UserRole.operator)
async def ansible_link(host_id: str):
"""Link, unlink, or create an AnsibleTarget from this host."""
from steward.models.ansible_inventory import AnsibleTarget
form = await request.form
action = form.get("action", "")
async with current_app.db_sessionmaker() as db:
async with db.begin():
if action == "unlink":
linked_result = await db.execute(
select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
)
linked = linked_result.scalar_one_or_none()
if linked:
linked.host_id = None
elif action == "link":
target_id = form.get("target_id", "").strip()
if target_id:
result = await db.execute(
select(AnsibleTarget).where(AnsibleTarget.id == target_id)
)
target = result.scalar_one_or_none()
if target:
# Clear any previous link for this host first
old_result = await db.execute(
select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
)
old = old_result.scalar_one_or_none()
if old and old.id != target_id:
old.host_id = None
target.host_id = host_id
elif action == "create":
host_result = await db.execute(select(Host).where(Host.id == host_id))
host = host_result.scalar_one_or_none()
if host:
new_target = AnsibleTarget(
id=str(uuid.uuid4()),
name=host.name,
address=host.address,
host_id=host_id,
)
db.add(new_target)
return redirect(f"/hosts/{host_id}")
@hosts_bp.post("/<host_id>/run-playbook")
@require_role(UserRole.operator)
async def run_playbook(host_id: str):
"""Run an Ansible playbook against just this host (ephemeral one-host inventory)."""
form = await request.form
async with current_app.db_sessionmaker() as db:
host = (await db.execute(select(Host).where(Host.id == host_id))).scalar_one_or_none()
if host is None:
return "Not found", 404
source_name = (form.get("source_name", "") or "").strip()
playbook = (form.get("playbook_path", "") or "").strip()
if not source_name or not playbook:
return "source and playbook are required", 400
try:
srcs = ansible_src.get_sources(current_app.config.get("ANSIBLE", {}))
except Exception:
return "Ansible sources are misconfigured", 400
source = next((s for s in srcs if s["name"] == source_name), None)
if source is None:
return "Source not found", 404
if playbook not in ansible_src.discover_playbooks(source["path"]):
return f"Playbook '{playbook}' not found in source '{source_name}'", 404
# Shared parser: discovered var__ fields → extra_vars_map, secret__ → secret_vars
# (unpersisted), plus tags/check. No --limit — there's exactly one host.
from steward.ansible.routes import _parse_run_params
params_or_none, secret_vars, err = _parse_run_params(form)
if err:
return err, 400
run_id = str(uuid.uuid4())
inv_content = ansible_src.host_inventory_content(host)
run = AnsibleRun(
id=run_id,
playbook_path=playbook,
inventory_path=f"host: {host.name}",
source_name=source_name,
triggered_by=session.get("user_id"),
status=AnsibleRunStatus.running,
params=params_or_none,
)
async with current_app.db_sessionmaker() as db:
async with db.begin():
db.add(run)
task = asyncio.create_task(
executor.start_run(
current_app._get_current_object(), # type: ignore[attr-defined]
run_id, playbook, f"host: {host.name}", source["path"],
params_or_none, inv_content, secret_vars=secret_vars,
)
)
task.add_done_callback(
lambda t: t.exception() and current_app.logger.error(
"Host playbook run %s raised: %s", run_id, t.exception()))
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"ansible.host_run", entity_type="host", entity_id=host.name,
detail={"playbook": playbook, "source": source_name})
return redirect(url_for("ansible.run_detail", run_id=run_id))
@hosts_bp.post("/<host_id>/delete")
@require_role(UserRole.admin)
async def delete_host(host_id: str):
host_name = None
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(select(Host).where(Host.id == host_id))
host = result.scalar_one_or_none()
if host:
host_name = host.name
await db.delete(host)
if host_name:
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"host.deleted", entity_type="host", entity_id=host_name)
return redirect(url_for("hosts.list_hosts"))
@hosts_bp.get("/uptime")
@require_role(UserRole.viewer)
async def uptime_page():
async with current_app.db_sessionmaker() as db:
result = await db.execute(select(Host).order_by(Host.name))
hosts = result.scalars().all()
uptime = await _compute_uptime(db)
return await render_template("hosts/uptime.html", hosts=hosts, uptime=uptime)
@hosts_bp.get("/uptime/widget")
@require_role(UserRole.viewer)
async def uptime_widget():
share_token = request.args.get("s")
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(Host).where(Host.ping_enabled.is_(True)).order_by(Host.name)
)
hosts = result.scalars().all()
uptime = await _compute_uptime(db)
return await render_template(
"hosts/uptime_widget.html",
hosts=hosts,
uptime=uptime,
share_token=share_token,
)