bvandeusenandClaude Opus 5.5
22b0be2b6c
#5767 "Steward hands a host a maintenance job and follows it to the end, through its own restarts", milestone 514. A host_maintenance_jobs row per job: queued → updating → [draining →] rebooting → [undraining →] done, or failed / timed_out. Steward's answer to an ingest post carries the step that host's agent should carry out now; the agent (1.18.0) leaves each step id for the root helper once, and the helper refuses an id it already ran, so repeats are harmless. Results come back as the #5766 maintenance events, matched by step id; the reboot completes when the host's `started` event has a boot_at after the reboot was handed out. The job reboots only when the update says one is needed (or never, if asked). A swarm node is drained first by itself if it's a manager or by a manager with the helper, and made active after; with no manager to drain it the job is refused. A minute tick times out a job past its state's deadline. A maintenance window over host_agent/<host> lasts the rest of the job. Everything is on the row, so a Steward restart, or the reboot of the host Steward runs on, resumes from the next post. Ingest runs it in its own savepoint so a job going wrong never costs the samples. The Agent panel leads with a running job and says what a finished one did; Manage agent has Update now (with "never reboot"), or says why a job can't start. Integration test: hand off, Steward restarts, the post-boot events close the job. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
311 lines
18 KiB
Python
311 lines
18 KiB
Python
"""The host page: what sits where (#5214)."""
|
||
import re
|
||
from types import SimpleNamespace
|
||
|
||
from quart import render_template, session
|
||
|
||
from steward.hosts.service import HOST_STATES, NO_GROUP
|
||
|
||
|
||
def _list_ctx(**over):
|
||
"""What the Hosts list route hands its template (#5294), with no filter set."""
|
||
ctx = dict(hosts=[], host_status={}, uptime={}, agent={}, groups={}, group_names=[],
|
||
f={"q": "", "group": "", "state": "", "sort": "status", "dir": "asc"},
|
||
filter_url=lambda **kw: "/hosts/?" + "&".join(f"{k}={v}" for k, v in kw.items()),
|
||
sort_url=lambda key: f"/hosts/?sort={key}",
|
||
states=HOST_STATES, no_group=NO_GROUP, add=None)
|
||
ctx.update(over)
|
||
ctx.setdefault("total", len(ctx["hosts"]))
|
||
return ctx
|
||
|
||
|
||
def _headers(page):
|
||
"""Each table header's text, tags and arrows stripped."""
|
||
return [re.sub(r"<[^>]+>|[↑↓]", "", h).strip() for h in re.findall(r"<th[^>]*>(.*?)</th>", page, re.S)]
|
||
|
||
|
||
async def test_ansible_comes_before_the_docker_list(app, monkeypatch):
|
||
# The Docker list can run to dozens of containers; Ansible under it was
|
||
# easy to miss, so it sits above.
|
||
monkeypatch.setitem(app.config, "PLUGINS", {"docker": {"enabled": True}})
|
||
host = SimpleNamespace(id="h1", name="box", address="10.0.0.5")
|
||
async with app.test_request_context("/hosts/h1"):
|
||
session["user_role"] = "operator"
|
||
page = await render_template(
|
||
"hosts/detail.html", host=host, monitors=[], uptime=None, alerts=[],
|
||
linked_target=None, linkable_targets=[], ansible_sources=[])
|
||
docker = page.index('hx-get="/plugins/docker/host/h1"')
|
||
# #5290: Monitors and Agent share a row whose cards drop their own margin.
|
||
assert '<div class="host-pair">' in page
|
||
assert page.index(">Ansible</h3>") < docker
|
||
assert page.index('hx-get="/plugins/host_agent/panel/h1"') < docker
|
||
# Monitors are added on their own form, filed under this host (#5277);
|
||
# each row's own page has the labelled Edit/Pause/Delete.
|
||
assert 'href="/monitors/new?host_id=h1"' in page
|
||
assert "row-actions" not in page and 'action="/monitors/add"' not in page
|
||
|
||
|
||
async def test_adding_a_monitor_from_a_host_page_says_whose_it_is(app):
|
||
host = SimpleNamespace(id="h1", name="box", address="10.0.0.5")
|
||
async with app.test_request_context("/monitors/new?host_id=h1"):
|
||
session["user_role"] = "operator"
|
||
page = await render_template(
|
||
"monitors/form.html", monitor=None, host=host, mtype="icmp", config={}, types=[],
|
||
values={"name": "", "target": "10.0.0.5", "host_id": "h1", "check_interval_seconds": 0})
|
||
assert '<a href="/hosts/h1">box</a>' in page # crumb and hint name the host
|
||
assert 'name="host_id" value="h1"' in page and 'value="10.0.0.5"' in page
|
||
assert 'href="/hosts/h1" class="btn btn-sm btn-ghost">Cancel' in page
|
||
|
||
|
||
async def test_the_agent_card_keeps_its_actions_under_manage(app):
|
||
"""#5278: "Remove" said nothing about what it removed."""
|
||
from datetime import datetime, timezone
|
||
from tests._templates import render_plugin_template
|
||
host = SimpleNamespace(id="h1", name="box", address="10.0.0.5")
|
||
reg = SimpleNamespace(agent_version="1.8.0", last_seen_at=datetime.now(timezone.utc))
|
||
target = SimpleNamespace(id="t1", name="box")
|
||
async with app.test_request_context("/plugins/host_agent/panel/h1"):
|
||
session["user_role"] = "admin"
|
||
page = await render_plugin_template(
|
||
app, "host_agent", "panel.html", host=host, reg=reg, reporting=True, stale=False,
|
||
target=target, ansible_available=True, managed_key_set=True)
|
||
manage = page.index(">Manage agent</summary>")
|
||
assert page.index("Full metrics →") < manage
|
||
for action in (">Update agent</button>", ">Rotate token</button>", ">Unregister agent</button>"):
|
||
assert page.index(action) > manage
|
||
assert ">Remove</button>" not in page
|
||
|
||
|
||
async def test_the_hosts_list_says_what_is_wrong_and_labels_its_figures(app):
|
||
"""#5279–#5281: labelled CPU and Memory, one uptime figure, status with how
|
||
long, no SLA tab and no per-row edit/delete."""
|
||
from datetime import datetime, timedelta, timezone
|
||
now = datetime.now(timezone.utc)
|
||
down = SimpleNamespace(id="h1", name="nas", address="10.0.0.5")
|
||
quiet = SimpleNamespace(id="h2", name="pi", address="")
|
||
async with app.test_request_context("/hosts/"):
|
||
session["user_role"] = "admin"
|
||
page = await render_template("hosts/list.html", **_list_ctx(
|
||
hosts=[down, quiet],
|
||
host_status={"h1": {"state": "down", "latency_ms": None, "count": 2,
|
||
"down_since": now - timedelta(minutes=12)}},
|
||
uptime={"h1": {"24h": 91.5, "7d": 98.0, "30d": 99.42}},
|
||
agent={"pi": {"fresh": False, "last_at": now - timedelta(minutes=7),
|
||
"cpu_pct": 3.0, "mem_used_pct": 41.0}}))
|
||
# No inventory groups: no Groups column.
|
||
assert _headers(page) == ["Name", "Address", "Status", "Latency", "Monitors", "30d uptime", "CPU", "Memory"]
|
||
assert "for 12m" in page and "agent silent" in page and "for 7m" in page
|
||
assert "99.42%" in page and 'title="24h 91.50% · 7d 98.00%"' in page
|
||
assert ">41%<" in page and "tone-dim" in page # a silent agent's last reading, greyed
|
||
assert "/hosts/uptime" not in page and ">Delete<" not in page and ">Edit<" not in page
|
||
|
||
|
||
async def test_a_host_without_an_agent_can_get_a_manual_install(app):
|
||
"""#5282: the manual install moved here from the fleet's add-host form."""
|
||
from tests._templates import render_plugin_template
|
||
host = SimpleNamespace(id="h1", name="box", address="10.0.0.5")
|
||
async with app.test_request_context("/plugins/host_agent/panel/h1"):
|
||
session["user_role"] = "admin"
|
||
page = await render_plugin_template(
|
||
app, "host_agent", "panel.html", host=host, reg=None, reporting=False, stale=False,
|
||
target=None, ansible_available=False, managed_key_set=False)
|
||
assert 'action="/plugins/host_agent/fleet/h1/register"' in page and ">Install manually</button>" in page
|
||
|
||
|
||
async def test_add_host_is_a_dialog_on_the_hosts_list(app):
|
||
"""#5291: the form was two fields on a page of its own."""
|
||
async with app.test_request_context("/hosts/"):
|
||
session["user_role"] = "operator"
|
||
closed = await render_template("hosts/list.html", **_list_ctx())
|
||
dialog = re.search(r'<dialog id="add-host".*?</dialog>', closed, re.S).group(0)
|
||
assert 'action="/hosts/"' in dialog and ">Cancel</button>" in dialog
|
||
assert re.search(r'<input[^>]*name="address"[^>]*>', dialog).group(0).count("required") == 0
|
||
assert "showModal()\">Add host</button>" in closed
|
||
assert "getElementById('add-host').showModal();</script>" not in closed
|
||
assert "/hosts/new" not in closed
|
||
|
||
async with app.test_request_context("/hosts/"):
|
||
session["user_role"] = "viewer"
|
||
viewer = await render_template("hosts/list.html", **_list_ctx(add={}))
|
||
assert 'id="add-host"' not in viewer and ">Add host<" not in viewer
|
||
|
||
|
||
async def test_add_host_sends_a_clash_back_with_what_was_typed(app):
|
||
"""#5283, #5291: the list again, the dialog open, an inline error and the typed values."""
|
||
async with app.test_request_context("/hosts/"):
|
||
session["user_role"] = "operator"
|
||
page = await render_template("hosts/list.html", **_list_ctx(
|
||
add={"values": {"name": "nas", "address": "10.0.0.9"}, "error": "A host named nas already exists."}))
|
||
assert 'role="alert">A host named nas already exists.<' in page
|
||
assert 'value="nas"' in page and 'value="10.0.0.9"' in page
|
||
assert "getElementById('add-host').showModal();</script>" in page
|
||
|
||
|
||
async def test_edit_host_keeps_its_page(app):
|
||
host = SimpleNamespace(id="h1", name="nas", address="10.0.0.5")
|
||
async with app.test_request_context("/hosts/h1/edit"):
|
||
session["user_role"] = "operator"
|
||
page = await render_template("hosts/form.html", host=host)
|
||
assert 'action="/hosts/h1"' in page and 'value="nas"' in page and 'value="10.0.0.5"' in page
|
||
assert '<a href="/hosts/h1">nas</a>' in page and ">Save</button>" in page
|
||
|
||
|
||
async def test_the_hosts_list_has_a_filter_bar_sortable_headers_and_group_chips(app):
|
||
"""#5294: search, group and status filters; headers that sort; a host's
|
||
groups as chips that filter the list."""
|
||
swarm = SimpleNamespace(id="h1", name="docker02", address="10.0.0.12")
|
||
lone = SimpleNamespace(id="h2", name="util01", address="10.0.0.9")
|
||
async with app.test_request_context("/hosts/"):
|
||
session["user_role"] = "viewer"
|
||
page = await render_template("hosts/list.html", **_list_ctx(
|
||
hosts=[swarm, lone], groups={"h1": ["Docker Swarm"]}, group_names=["Docker Swarm"],
|
||
f={"q": "", "group": "", "state": "", "sort": "cpu", "dir": "desc"}))
|
||
bar = re.search(r'<form class="filter-bar".*?</form>', page, re.S).group(0)
|
||
assert 'type="search" name="q"' in bar and 'hx-target="#hosts-table"' in bar and 'hx-push-url="true"' in bar
|
||
assert '<option value="Docker Swarm">' in bar and f'<option value="{NO_GROUP}">No group</option>' in bar
|
||
assert [o for o in re.findall(r'<option value="(\w*)"', bar.split('name="state"')[1])] == ["", "down", "silent", "up", "none"]
|
||
assert '<input type="hidden" name="sort" value="cpu">' in bar # a filter keeps the column sort
|
||
assert _headers(page) == ["Name", "Address", "Groups", "Status", "Latency", "Monitors", "30d uptime", "CPU", "Memory"]
|
||
assert 'aria-sort="descending"' in page and page.count('aria-sort="none"') == 5
|
||
assert 'href="/hosts/?sort=uptime"' in page
|
||
assert '<a class="chip" href="/hosts/?group=Docker Swarm"' in page
|
||
table = page[page.index('<div id="hosts-table">'):]
|
||
assert table.index("docker02") < table.index("util01")
|
||
|
||
|
||
def test_filter_hosts_by_search_group_and_status():
|
||
from steward.hosts.service import filter_hosts
|
||
hosts = [SimpleNamespace(id="h1", name="Docker02", address="10.0.0.12"),
|
||
SimpleNamespace(id="h2", name="util01", address="10.0.0.9"),
|
||
SimpleNamespace(id="h3", name="nas", address="")]
|
||
status = {"h1": {"state": "down"}, "h2": {"state": "up"}}
|
||
agent = {"util01": {"fresh": False}}
|
||
groups = {"h1": ["Docker Swarm", "Linux"], "h2": ["Linux"]}
|
||
|
||
def names(**kw):
|
||
return [h.name for h in filter_hosts(hosts, status=status, agent=agent, groups=groups, **kw)]
|
||
|
||
assert names() == ["Docker02", "util01", "nas"]
|
||
assert names(q="docker") == ["Docker02"] and names(q="0.0.9") == ["util01"]
|
||
assert names(group="Linux") == ["Docker02", "util01"] # membership overlaps
|
||
assert names(group=NO_GROUP) == ["nas"]
|
||
assert names(state="down") == ["Docker02"] and names(state="none") == ["nas"]
|
||
assert names(state="silent") == ["util01"] and names(state="up") == ["util01"]
|
||
assert names(group="Linux", state="up") == ["util01"]
|
||
|
||
|
||
def test_sort_hosts_puts_missing_values_last_either_way():
|
||
from steward.hosts.service import sort_hosts
|
||
hosts = [SimpleNamespace(id=i, name=n, address="") for i, n in [("a", "alpha"), ("b", "bravo"), ("c", "charlie")]]
|
||
agent = {"alpha": {"cpu_pct": 10.0, "fresh": True}, "bravo": {"cpu_pct": 70.0, "fresh": True}}
|
||
status = {"c": {"state": "down"}}
|
||
kw = dict(status=status, agent=agent, uptime={})
|
||
|
||
def names(sort, desc):
|
||
return [h.name for h in sort_hosts(hosts, sort, desc, **kw)]
|
||
|
||
assert names("cpu", True) == ["bravo", "alpha", "charlie"]
|
||
assert names("cpu", False) == ["alpha", "bravo", "charlie"] # no reading: last both ways
|
||
assert names("name", True) == ["charlie", "bravo", "alpha"]
|
||
assert names("status", False)[0] == "charlie" # down first
|
||
|
||
|
||
async def test_the_agent_card_says_what_happened_while_the_host_was_away(app):
|
||
"""#5756: a reboot reads as a reboot with its length, one still going says so."""
|
||
from datetime import datetime, timedelta, timezone
|
||
from tests._templates import render_plugin_template
|
||
now = datetime.now(timezone.utc)
|
||
host = SimpleNamespace(id="h1", name="box", address="10.0.0.5")
|
||
reg = SimpleNamespace(agent_version="1.14.0", last_seen_at=now)
|
||
downtime = [
|
||
{"kind": "reboot", "label": "Rebooted", "started_at": now - timedelta(minutes=5),
|
||
"ended_at": None, "ongoing": True, "seconds": 300.0},
|
||
{"kind": "lost_contact", "label": "Lost contact", "started_at": now - timedelta(hours=3),
|
||
"ended_at": now - timedelta(hours=2), "ongoing": False, "seconds": 3600.0},
|
||
]
|
||
async with app.test_request_context("/plugins/host_agent/panel/h1"):
|
||
session["user_role"] = "viewer"
|
||
page = await render_plugin_template(
|
||
app, "host_agent", "panel.html", host=host, reg=reg, reporting=True, stale=False,
|
||
target=None, ansible_available=False, managed_key_set=False, downtime=downtime)
|
||
quiet = await render_plugin_template(
|
||
app, "host_agent", "panel.html", host=host, reg=reg, reporting=True, stale=False,
|
||
target=None, ansible_available=False, managed_key_set=False, downtime=[])
|
||
assert "Downtime, last 7 days" in page
|
||
assert page.index("Rebooted") < page.index("Lost contact")
|
||
assert "going on, 5m" in page and "1h" in page
|
||
assert "Downtime, last 7 days" not in quiet
|
||
|
||
|
||
async def test_the_agent_card_leads_with_a_needed_reboot_then_the_updates(app):
|
||
"""#5765: what the host has waiting, from the agent's hourly check."""
|
||
from datetime import datetime, timedelta, timezone
|
||
from tests._templates import render_plugin_template
|
||
now = datetime.now(timezone.utc)
|
||
host = SimpleNamespace(id="h1", name="box", address="10.0.0.5")
|
||
waiting = SimpleNamespace(
|
||
agent_version="1.16.0", last_seen_at=now, updates_manager="apt", updates_pending=12,
|
||
updates_security=3, updates_checked_at=now - timedelta(minutes=20), updates_error=None,
|
||
reboot_required=True, reboot_required_since=now - timedelta(hours=5),
|
||
reboot_packages="libc6, linux-image-6.8.0-146-generic")
|
||
current = SimpleNamespace(**{**vars(waiting), "updates_pending": 0, "updates_security": 0,
|
||
"reboot_required": False, "reboot_required_since": None,
|
||
"reboot_packages": None})
|
||
kw = dict(host=host, reporting=True, stale=False, target=None, ansible_available=False,
|
||
managed_key_set=False, downtime=[])
|
||
async with app.test_request_context("/plugins/host_agent/panel/h1"):
|
||
session["user_role"] = "viewer"
|
||
page = await render_plugin_template(app, "host_agent", "panel.html", reg=waiting, **kw)
|
||
quiet = await render_plugin_template(app, "host_agent", "panel.html", reg=current, **kw)
|
||
assert page.index("Reboot required") < page.index("12 updates waiting (3 security)")
|
||
assert "libc6" in page
|
||
assert "Up to date" in quiet and "Reboot required" not in quiet
|
||
|
||
|
||
async def test_manage_agent_says_whether_the_maintenance_helper_is_installed(app):
|
||
"""#5766: the helper arrives with Update agent; until then the panel says so."""
|
||
from datetime import datetime, timezone
|
||
from tests._templates import render_plugin_template
|
||
host = SimpleNamespace(id="h1", name="box", address="10.0.0.5")
|
||
reg = SimpleNamespace(agent_version="1.17.0", last_seen_at=datetime.now(timezone.utc),
|
||
maint_helper=True)
|
||
kw = dict(host=host, reporting=True, stale=False, target=None, ansible_available=False,
|
||
managed_key_set=False, downtime=[])
|
||
async with app.test_request_context("/plugins/host_agent/panel/h1"):
|
||
session["user_role"] = "admin"
|
||
installed = await render_plugin_template(app, "host_agent", "panel.html", reg=reg, **kw)
|
||
missing = await render_plugin_template(
|
||
app, "host_agent", "panel.html",
|
||
reg=SimpleNamespace(**{**vars(reg), "maint_helper": None}), **kw)
|
||
assert "Maintenance helper installed" in installed
|
||
assert "No maintenance helper yet: Update agent installs it" in missing
|
||
|
||
|
||
async def test_the_agent_card_follows_a_maintenance_job_and_says_why_it_cant_start_one(app):
|
||
"""#5767: a running job leads what the host has waiting; Update now is
|
||
offered only when a job can start, else the reason is shown."""
|
||
from datetime import datetime, timedelta, timezone
|
||
from plugins.host_agent.maintenance import summary
|
||
from tests._templates import render_plugin_template
|
||
now = datetime.now(timezone.utc)
|
||
host = SimpleNamespace(id="h1", name="box", address="10.0.0.5")
|
||
reg = SimpleNamespace(agent_version="1.18.0", last_seen_at=now, maint_helper=True,
|
||
updates_checked_at=now, updates_pending=4, updates_security=0,
|
||
reboot_required=False)
|
||
job = SimpleNamespace(state="rebooting", state_at=now - timedelta(minutes=2), finished_at=None,
|
||
detail=None, packages=4, back_at=None, reboot_needed=True,
|
||
reboot_policy="if_required")
|
||
kw = dict(host=host, reg=reg, reporting=True, stale=False, target=None, ansible_available=False,
|
||
managed_key_set=False, downtime=[])
|
||
async with app.test_request_context("/plugins/host_agent/panel/h1"):
|
||
session["user_role"] = "admin"
|
||
busy = await render_plugin_template(
|
||
app, "host_agent", "panel.html", maint_job=job, maint_summary=summary(job),
|
||
maint_blocker="a maintenance job is already running on it", **kw)
|
||
free = await render_plugin_template(app, "host_agent", "panel.html", maint_job=None,
|
||
maint_blocker=None, **kw)
|
||
assert busy.index("Maintenance: Rebooting") < busy.index("4 updates waiting")
|
||
assert "Can't update from here: a maintenance job is already running on it." in busy
|
||
assert ">Update now</button>" not in busy
|
||
assert ">Update now</button>" in free and 'name="reboot_policy" value="never"' in free
|