988c13d51f
Milestone 72 phase C — bring the host-view graphs onto the dashboard:
- host_resource_history widget reworked into a real host-view chart: epoch-ms
linear axis (no Chart.js date adapter), themed like the host-detail charts,
maintainAspectRatio:false so it fills the resized panel, unique canvas per
widget instance (wid), and empty states ("pick a host" / "no metrics yet").
Was previously unusable — it had a broken time axis and no way to choose a host.
- Add a "host" param type: the edit form renders a live dropdown of hosts
(dashboard routes now pass the host list to the editor); the chosen host_id is
stored in config and fed to the widget.
- Hosts-overview widget gains a per-row CPU sparkline (last hour) via the shared
sparkline_svg helper — the host-view at-a-glance trend, on the main widget.
Charts/sparklines render only in the browser, so CI can't exercise them — needs
an operator visual check.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
529 lines
21 KiB
Python
529 lines
21 KiB
Python
from __future__ import annotations
|
|
import asyncio
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
|
|
from sqlalchemy import and_, case, select, func
|
|
from steward.ansible import executor, sources as ansible_src
|
|
from steward.auth.middleware import require_role
|
|
from steward.core.audit import log_audit
|
|
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
|
from steward.models.hosts import Host, ProbeType
|
|
from steward.models.monitors import PingResult, DnsResult, PingStatus
|
|
from steward.models.users import UserRole
|
|
|
|
hosts_bp = Blueprint("hosts", __name__, url_prefix="/hosts")
|
|
|
|
|
|
def _ansible_source_names() -> list[str]:
|
|
try:
|
|
return [s["name"] for s in ansible_src.get_sources(current_app.config.get("ANSIBLE", {}))]
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
async def _compute_uptime(db) -> dict[str, dict]:
|
|
"""Return per-host uptime % for 24h, 7d, 30d windows.
|
|
|
|
Returns {host_id: {"24h": float|None, "7d": float|None, "30d": float|None}}.
|
|
"""
|
|
now = datetime.now(timezone.utc)
|
|
cutoff_30d = now - timedelta(days=30)
|
|
cutoff_7d = now - timedelta(days=7)
|
|
cutoff_24h = now - timedelta(hours=24)
|
|
|
|
result = await db.execute(
|
|
select(
|
|
PingResult.host_id,
|
|
# 30d window — all rows in query qualify
|
|
func.count(PingResult.id).label("total_30d"),
|
|
func.sum(case((PingResult.status == PingStatus.up, 1), else_=0)).label("up_30d"),
|
|
# 7d sub-window
|
|
func.sum(case((PingResult.probed_at >= cutoff_7d, 1), else_=0)).label("total_7d"),
|
|
func.sum(case(
|
|
(and_(PingResult.probed_at >= cutoff_7d, PingResult.status == PingStatus.up), 1),
|
|
else_=0,
|
|
)).label("up_7d"),
|
|
# 24h sub-window
|
|
func.sum(case((PingResult.probed_at >= cutoff_24h, 1), else_=0)).label("total_24h"),
|
|
func.sum(case(
|
|
(and_(PingResult.probed_at >= cutoff_24h, PingResult.status == PingStatus.up), 1),
|
|
else_=0,
|
|
)).label("up_24h"),
|
|
)
|
|
.where(PingResult.probed_at >= cutoff_30d)
|
|
.group_by(PingResult.host_id)
|
|
)
|
|
|
|
def _pct(up, total):
|
|
return round(float(up) / float(total) * 100, 2) if total else None
|
|
|
|
stats: dict[str, dict] = {}
|
|
for row in result:
|
|
stats[row.host_id] = {
|
|
"24h": _pct(row.up_24h, row.total_24h),
|
|
"7d": _pct(row.up_7d, row.total_7d),
|
|
"30d": _pct(row.up_30d, row.total_30d),
|
|
}
|
|
return stats
|
|
|
|
|
|
async def _agent_metrics_by_host(db, host_names: list[str]) -> dict[str, dict]:
|
|
"""Latest agent cpu/mem per host name from the generic PluginMetric table.
|
|
|
|
Read directly (PluginMetric is a core model) so the Hosts list can show an
|
|
at-a-glance agent column without importing the host_agent plugin.
|
|
"""
|
|
from steward.models.metrics import PluginMetric
|
|
if not host_names:
|
|
return {}
|
|
wanted = ("cpu_pct", "mem_used_pct")
|
|
subq = (
|
|
select(
|
|
PluginMetric.resource_name, PluginMetric.metric_name,
|
|
func.max(PluginMetric.recorded_at).label("m"),
|
|
)
|
|
.where(
|
|
PluginMetric.source_module == "host_agent",
|
|
PluginMetric.metric_name.in_(wanted),
|
|
PluginMetric.resource_name.in_(host_names),
|
|
)
|
|
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
|
|
).subquery()
|
|
rows = (await db.execute(
|
|
select(PluginMetric).join(
|
|
subq,
|
|
(PluginMetric.resource_name == subq.c.resource_name)
|
|
& (PluginMetric.metric_name == subq.c.metric_name)
|
|
& (PluginMetric.recorded_at == subq.c.m),
|
|
)
|
|
)).scalars().all()
|
|
out: dict[str, dict] = {}
|
|
for r in rows:
|
|
out.setdefault(r.resource_name, {})[r.metric_name] = r.value
|
|
return out
|
|
|
|
|
|
async def _agent_overview_by_host(db, host_names: list[str]) -> dict[str, dict]:
|
|
"""Per-host agent glance (cpu/mem/root-disk + freshness) from PluginMetric.
|
|
|
|
Core-safe (no host_agent import). Freshness compares the latest host-level
|
|
sample against the plugin's stale window (config), defaulting to 180s.
|
|
"""
|
|
from steward.models.metrics import PluginMetric
|
|
if not host_names:
|
|
return {}
|
|
now = datetime.now(timezone.utc)
|
|
stale_after = int(
|
|
(current_app.config.get("PLUGINS", {}).get("host_agent", {}) or {})
|
|
.get("stale_after_seconds", 180))
|
|
resources = list(host_names) + [h + ":/" for h in host_names]
|
|
subq = (
|
|
select(
|
|
PluginMetric.resource_name, PluginMetric.metric_name,
|
|
func.max(PluginMetric.recorded_at).label("m"),
|
|
)
|
|
.where(
|
|
PluginMetric.source_module == "host_agent",
|
|
PluginMetric.resource_name.in_(resources),
|
|
PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "disk_used_pct")),
|
|
)
|
|
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
|
|
).subquery()
|
|
rows = (await db.execute(
|
|
select(PluginMetric).join(
|
|
subq,
|
|
(PluginMetric.resource_name == subq.c.resource_name)
|
|
& (PluginMetric.metric_name == subq.c.metric_name)
|
|
& (PluginMetric.recorded_at == subq.c.m),
|
|
).where(PluginMetric.source_module == "host_agent")
|
|
)).scalars().all()
|
|
out: dict[str, dict] = {}
|
|
for r in rows:
|
|
if r.resource_name.endswith(":/") and r.metric_name == "disk_used_pct":
|
|
out.setdefault(r.resource_name[:-2], {})["disk_root"] = r.value
|
|
elif r.resource_name in host_names:
|
|
d = out.setdefault(r.resource_name, {})
|
|
d[r.metric_name] = r.value
|
|
if d.get("_ts") is None or r.recorded_at > d["_ts"]:
|
|
d["_ts"] = r.recorded_at
|
|
for d in out.values():
|
|
ts = d.pop("_ts", None)
|
|
d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
|
|
return out
|
|
|
|
|
|
async def _agent_cpu_sparks_by_host(db, host_names: list[str], hours: int = 1) -> dict[str, str]:
|
|
"""{host_name: inline-SVG cpu sparkline} over the last `hours` (core-safe).
|
|
|
|
A tiny at-a-glance trend in each row — the host-view sparkline, on the widget.
|
|
"""
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.core.status import sparkline_svg
|
|
if not host_names:
|
|
return {}
|
|
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
|
|
rows = (await db.execute(
|
|
select(PluginMetric.resource_name, PluginMetric.value, PluginMetric.recorded_at)
|
|
.where(
|
|
PluginMetric.source_module == "host_agent",
|
|
PluginMetric.metric_name == "cpu_pct",
|
|
PluginMetric.resource_name.in_(host_names),
|
|
PluginMetric.recorded_at >= cutoff,
|
|
)
|
|
.order_by(PluginMetric.recorded_at)
|
|
)).all()
|
|
by_host: dict[str, list[float]] = {}
|
|
for r in rows:
|
|
by_host.setdefault(r.resource_name, []).append(r.value)
|
|
# Amber line to read as "CPU"; only draw when there are a couple of points.
|
|
return {
|
|
name: sparkline_svg(vals[-40:], width=64, height=18, stroke="#c8a840")
|
|
for name, vals in by_host.items() if len(vals) >= 2
|
|
}
|
|
|
|
|
|
@hosts_bp.get("/overview/widget")
|
|
@require_role(UserRole.viewer)
|
|
async def overview_widget():
|
|
"""Dashboard widget: unified per-host monitor + agent glance, linking to the hub."""
|
|
async with current_app.db_sessionmaker() as db:
|
|
hosts = (await db.execute(select(Host).order_by(Host.name))).scalars().all()
|
|
latest_ping_subq = (
|
|
select(PingResult.host_id, func.max(PingResult.probed_at).label("max_at"))
|
|
.group_by(PingResult.host_id).subquery()
|
|
)
|
|
latest_pings = {r.host_id: r for r in (await db.execute(
|
|
select(PingResult).join(
|
|
latest_ping_subq,
|
|
(PingResult.host_id == latest_ping_subq.c.host_id)
|
|
& (PingResult.probed_at == latest_ping_subq.c.max_at),
|
|
))).scalars()}
|
|
latest_dns_subq = (
|
|
select(DnsResult.host_id, func.max(DnsResult.resolved_at).label("max_at"))
|
|
.group_by(DnsResult.host_id).subquery()
|
|
)
|
|
latest_dns = {r.host_id: r for r in (await db.execute(
|
|
select(DnsResult).join(
|
|
latest_dns_subq,
|
|
(DnsResult.host_id == latest_dns_subq.c.host_id)
|
|
& (DnsResult.resolved_at == latest_dns_subq.c.max_at),
|
|
))).scalars()}
|
|
uptime = await _compute_uptime(db)
|
|
host_names = [h.name for h in hosts]
|
|
agent = await _agent_overview_by_host(db, host_names)
|
|
cpu_sparks = await _agent_cpu_sparks_by_host(db, host_names)
|
|
|
|
return await render_template(
|
|
"hosts/overview_widget.html",
|
|
hosts=hosts, latest_pings=latest_pings, latest_dns=latest_dns,
|
|
uptime=uptime, agent=agent, cpu_sparks=cpu_sparks,
|
|
)
|
|
|
|
|
|
@hosts_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def list_hosts():
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(select(Host).order_by(Host.name))
|
|
hosts = result.scalars().all()
|
|
|
|
# Latest ping result per host
|
|
latest_ping_subq = (
|
|
select(PingResult.host_id, func.max(PingResult.probed_at).label("max_at"))
|
|
.group_by(PingResult.host_id)
|
|
.subquery()
|
|
)
|
|
pr = await db.execute(
|
|
select(PingResult).join(
|
|
latest_ping_subq,
|
|
(PingResult.host_id == latest_ping_subq.c.host_id)
|
|
& (PingResult.probed_at == latest_ping_subq.c.max_at),
|
|
)
|
|
)
|
|
latest_pings = {r.host_id: r for r in pr.scalars()}
|
|
|
|
# Latest DNS result per host
|
|
latest_dns_subq = (
|
|
select(DnsResult.host_id, func.max(DnsResult.resolved_at).label("max_at"))
|
|
.group_by(DnsResult.host_id)
|
|
.subquery()
|
|
)
|
|
dr = await db.execute(
|
|
select(DnsResult).join(
|
|
latest_dns_subq,
|
|
(DnsResult.host_id == latest_dns_subq.c.host_id)
|
|
& (DnsResult.resolved_at == latest_dns_subq.c.max_at),
|
|
)
|
|
)
|
|
latest_dns = {r.host_id: r for r in dr.scalars()}
|
|
uptime = await _compute_uptime(db)
|
|
agent_metrics = await _agent_metrics_by_host(db, [h.name for h in hosts])
|
|
|
|
return await render_template(
|
|
"hosts/list.html",
|
|
hosts=hosts,
|
|
latest_pings=latest_pings,
|
|
latest_dns=latest_dns,
|
|
uptime=uptime,
|
|
agent_metrics=agent_metrics,
|
|
)
|
|
|
|
|
|
@hosts_bp.get("/<host_id>")
|
|
@require_role(UserRole.viewer)
|
|
async def host_detail(host_id: str):
|
|
"""The host hub: monitors + agent (embedded fragment) + Ansible, one view."""
|
|
from steward.models.ansible_inventory import AnsibleTarget
|
|
from sqlalchemy.orm import selectinload
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
host = (await db.execute(
|
|
select(Host).where(Host.id == host_id))).scalar_one_or_none()
|
|
if host is None:
|
|
return "Not found", 404
|
|
ping = (await db.execute(
|
|
select(PingResult).where(PingResult.host_id == host_id)
|
|
.order_by(PingResult.probed_at.desc()).limit(1))).scalar_one_or_none()
|
|
dns = (await db.execute(
|
|
select(DnsResult).where(DnsResult.host_id == host_id)
|
|
.order_by(DnsResult.resolved_at.desc()).limit(1))).scalar_one_or_none()
|
|
uptime = (await _compute_uptime(db)).get(host_id)
|
|
linked_target = (await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
|
|
.options(selectinload(AnsibleTarget.groups)))).scalar_one_or_none()
|
|
linkable_targets = (await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id.is_(None))
|
|
.order_by(AnsibleTarget.name))).scalars().all()
|
|
|
|
return await render_template(
|
|
"hosts/detail.html",
|
|
host=host, ping=ping, dns=dns, uptime=uptime,
|
|
linked_target=linked_target, linkable_targets=linkable_targets,
|
|
ansible_sources=_ansible_source_names(),
|
|
)
|
|
|
|
|
|
@hosts_bp.get("/new")
|
|
@require_role(UserRole.operator)
|
|
async def new_host():
|
|
return await render_template("hosts/form.html", host=None, probe_types=list(ProbeType))
|
|
|
|
|
|
@hosts_bp.post("/")
|
|
@require_role(UserRole.operator)
|
|
async def create_host():
|
|
form = await request.form
|
|
host = Host(
|
|
name=form["name"].strip(),
|
|
address=form["address"].strip(),
|
|
probe_type=ProbeType(form.get("probe_type", "tcp")),
|
|
probe_port=int(form.get("probe_port") or 80),
|
|
ping_enabled="ping_enabled" in form,
|
|
dns_enabled="dns_enabled" in form,
|
|
dns_expected_ip=form.get("dns_expected_ip", "").strip() or None,
|
|
# poll_interval_seconds not exposed in UI yet; per-host scheduling not implemented
|
|
)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
db.add(host)
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"host.created", entity_type="host", entity_id=host.name,
|
|
detail={"address": host.address})
|
|
return redirect(url_for("hosts.list_hosts"))
|
|
|
|
|
|
@hosts_bp.get("/<host_id>/edit")
|
|
@require_role(UserRole.operator)
|
|
async def edit_host(host_id: str):
|
|
# Pure host config (name/address/monitors). Agent, Ansible target linking,
|
|
# and playbook runs live on the host detail page (the hub), not here.
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(select(Host).where(Host.id == host_id))
|
|
host = result.scalar_one_or_none()
|
|
if host is None:
|
|
return "Not found", 404
|
|
|
|
return await render_template(
|
|
"hosts/form.html", host=host, probe_types=list(ProbeType))
|
|
|
|
|
|
@hosts_bp.post("/<host_id>")
|
|
@require_role(UserRole.operator)
|
|
async def update_host(host_id: str):
|
|
form = await request.form
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(Host).where(Host.id == host_id))
|
|
host = result.scalar_one_or_none()
|
|
if host is None:
|
|
return "Not found", 404
|
|
host.name = form["name"].strip()
|
|
host.address = form["address"].strip()
|
|
host.probe_type = ProbeType(form.get("probe_type", "tcp"))
|
|
host.probe_port = int(form.get("probe_port") or 80)
|
|
host.ping_enabled = "ping_enabled" in form
|
|
host.dns_enabled = "dns_enabled" in form
|
|
host.dns_expected_ip = form.get("dns_expected_ip", "").strip() or None
|
|
# poll_interval_seconds not exposed in UI yet
|
|
return redirect(url_for("hosts.list_hosts"))
|
|
|
|
|
|
@hosts_bp.post("/<host_id>/ansible-link")
|
|
@require_role(UserRole.operator)
|
|
async def ansible_link(host_id: str):
|
|
"""Link, unlink, or create an AnsibleTarget from this host."""
|
|
from steward.models.ansible_inventory import AnsibleTarget
|
|
|
|
form = await request.form
|
|
action = form.get("action", "")
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
if action == "unlink":
|
|
linked_result = await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
|
|
)
|
|
linked = linked_result.scalar_one_or_none()
|
|
if linked:
|
|
linked.host_id = None
|
|
|
|
elif action == "link":
|
|
target_id = form.get("target_id", "").strip()
|
|
if target_id:
|
|
result = await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.id == target_id)
|
|
)
|
|
target = result.scalar_one_or_none()
|
|
if target:
|
|
# Clear any previous link for this host first
|
|
old_result = await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
|
|
)
|
|
old = old_result.scalar_one_or_none()
|
|
if old and old.id != target_id:
|
|
old.host_id = None
|
|
target.host_id = host_id
|
|
|
|
elif action == "create":
|
|
host_result = await db.execute(select(Host).where(Host.id == host_id))
|
|
host = host_result.scalar_one_or_none()
|
|
if host:
|
|
new_target = AnsibleTarget(
|
|
id=str(uuid.uuid4()),
|
|
name=host.name,
|
|
address=host.address,
|
|
host_id=host_id,
|
|
)
|
|
db.add(new_target)
|
|
|
|
return redirect(f"/hosts/{host_id}")
|
|
|
|
|
|
@hosts_bp.post("/<host_id>/run-playbook")
|
|
@require_role(UserRole.operator)
|
|
async def run_playbook(host_id: str):
|
|
"""Run an Ansible playbook against just this host (ephemeral one-host inventory)."""
|
|
form = await request.form
|
|
async with current_app.db_sessionmaker() as db:
|
|
host = (await db.execute(select(Host).where(Host.id == host_id))).scalar_one_or_none()
|
|
if host is None:
|
|
return "Not found", 404
|
|
|
|
source_name = (form.get("source_name", "") or "").strip()
|
|
playbook = (form.get("playbook_path", "") or "").strip()
|
|
if not source_name or not playbook:
|
|
return "source and playbook are required", 400
|
|
try:
|
|
srcs = ansible_src.get_sources(current_app.config.get("ANSIBLE", {}))
|
|
except Exception:
|
|
return "Ansible sources are misconfigured", 400
|
|
source = next((s for s in srcs if s["name"] == source_name), None)
|
|
if source is None:
|
|
return "Source not found", 404
|
|
if playbook not in ansible_src.discover_playbooks(source["path"]):
|
|
return f"Playbook '{playbook}' not found in source '{source_name}'", 404
|
|
|
|
# Shared parser: discovered var__ fields → extra_vars_map, secret__ → secret_vars
|
|
# (unpersisted), plus tags/check. No --limit — there's exactly one host.
|
|
from steward.ansible.routes import _parse_run_params
|
|
params_or_none, secret_vars, err = _parse_run_params(form)
|
|
if err:
|
|
return err, 400
|
|
|
|
run_id = str(uuid.uuid4())
|
|
inv_content = ansible_src.host_inventory_content(host)
|
|
run = AnsibleRun(
|
|
id=run_id,
|
|
playbook_path=playbook,
|
|
inventory_path=f"host: {host.name}",
|
|
source_name=source_name,
|
|
triggered_by=session.get("user_id"),
|
|
status=AnsibleRunStatus.running,
|
|
params=params_or_none,
|
|
)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
db.add(run)
|
|
|
|
task = asyncio.create_task(
|
|
executor.start_run(
|
|
current_app._get_current_object(), # type: ignore[attr-defined]
|
|
run_id, playbook, f"host: {host.name}", source["path"],
|
|
params_or_none, inv_content, secret_vars=secret_vars,
|
|
)
|
|
)
|
|
task.add_done_callback(
|
|
lambda t: t.exception() and current_app.logger.error(
|
|
"Host playbook run %s raised: %s", run_id, t.exception()))
|
|
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"ansible.host_run", entity_type="host", entity_id=host.name,
|
|
detail={"playbook": playbook, "source": source_name})
|
|
return redirect(url_for("ansible.run_detail", run_id=run_id))
|
|
|
|
|
|
@hosts_bp.post("/<host_id>/delete")
|
|
@require_role(UserRole.admin)
|
|
async def delete_host(host_id: str):
|
|
host_name = None
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(Host).where(Host.id == host_id))
|
|
host = result.scalar_one_or_none()
|
|
if host:
|
|
host_name = host.name
|
|
await db.delete(host)
|
|
if host_name:
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"host.deleted", entity_type="host", entity_id=host_name)
|
|
return redirect(url_for("hosts.list_hosts"))
|
|
|
|
|
|
@hosts_bp.get("/uptime")
|
|
@require_role(UserRole.viewer)
|
|
async def uptime_page():
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(select(Host).order_by(Host.name))
|
|
hosts = result.scalars().all()
|
|
uptime = await _compute_uptime(db)
|
|
return await render_template("hosts/uptime.html", hosts=hosts, uptime=uptime)
|
|
|
|
|
|
@hosts_bp.get("/uptime/widget")
|
|
@require_role(UserRole.viewer)
|
|
async def uptime_widget():
|
|
share_token = request.args.get("s")
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(
|
|
select(Host).where(Host.ping_enabled.is_(True)).order_by(Host.name)
|
|
)
|
|
hosts = result.scalars().all()
|
|
uptime = await _compute_uptime(db)
|
|
return await render_template(
|
|
"hosts/uptime_widget.html",
|
|
hosts=hosts,
|
|
uptime=uptime,
|
|
share_token=share_token,
|
|
)
|