feat: the host agent is always on; the MCP sees and starts host maintenance (#5834)

#5834 "The host agent is always on, and the MCP sees a host's
maintenance", step 1 of milestone 520 "The MCP sees what Steward has
grown".

The host agent is a core system (operator, 2026-10-15: "it's very core
to the functionality of Steward as a whole and shouldn't be considered
something that can be toggled off"). ALWAYS_ON_PLUGINS in core settings
makes to_plugins_cfg report it enabled whatever its stored row says, so
it always loads and its MCP tools are always listed. Settings -> Plugins
shows it as "Built in: always on" with no switch, and a save keeps it
enabled. Its code and URLs stay under plugins/host_agent: every
installed agent posts to /plugins/host_agent/ingest.

Maintenance through the MCP:
- list_maintenance_plans: each plan's group, weekly slot, reboot policy,
  next slot, and its latest run (hosts, how far it got, who it skipped
  and why, how it ended).
- get_maintenance_job(host | job): a host's jobs, newest and running
  first, or one job: state, packages updated, reboot, drained by, why it
  failed, when its current state times out.
- run_maintenance(host, reboot="if_required" | "never"): admin, as the
  host page's Update now; refuses a busy host or one with no helper;
  audited "<user> via MCP".
- The overview names a host whose latest job in the last week failed or
  timed out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_settings_default_plugins.py

@@ -41,3 +41,14 @@ def test_defaults_use_plugin_dot_namespace():
     # The keys must live under the plugin.<name> namespace to_plugins_cfg reads.
     for name in DEFAULT_ON:
         assert f"plugin.{name}" in settings_module.DEFAULTS
+
+
+def test_the_host_agent_is_always_on():
+    """Steward relies on it, so a stored disable, or no row at all, still
+    loads it (operator, 2026-10-15); its other settings are kept."""
+    stored_off = {**DEFAULTS, "plugin.host_agent": {"enabled": False, "stale_after_seconds": 300}}
+    assert to_plugins_cfg(stored_off)["host_agent"] == {"enabled": True, "stale_after_seconds": 300}
+    no_row = {k: v for k, v in DEFAULTS.items() if k != "plugin.host_agent"}
+    assert to_plugins_cfg(no_row)["host_agent"]["enabled"] is True
+    # Only the host agent: another plugin's stored disable still wins.
+    assert to_plugins_cfg({**DEFAULTS, "plugin.snmp": {"enabled": False}})["snmp"]["enabled"] is False

tests/integration/test_mcp_maintenance.py

@@ -0,0 +1,156 @@
+"""Integration (#5834): the MCP's maintenance tools against real Postgres.
+
+A plan on a group, a host whose last job failed and one with a job running,
+read back through list_maintenance_plans and get_maintenance_job; run_maintenance
+refuses a busy host, starts one on a free host and audits it; the overview
+names the failed job.
+
+Rows are seeded with plain SQL so this file never imports the plugin's models
+(a second import after the app loaded the plugin raises "Table already
+defined"). The tool module comes from the loaded plugin.
+"""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import importlib
+import os
+import uuid
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def _tools():
+    from steward.core.plugin_manager import get_loaded_plugin_modules
+    loaded = get_loaded_plugin_modules().get("host_agent")
+    base = loaded.__name__ if loaded is not None else "plugins.host_agent"
+    return importlib.import_module(f"{base}.mcp_tools")
+
+
+def test_maintenance_plans_jobs_and_starting_one_through_mcp(app):
+    from sqlalchemy import select, text
+    from steward.core.api_tokens import Caller
+    from steward.mcp import _context
+    from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget
+    from steward.models.audit import AuditEvent
+    from steward.models.hosts import Host
+    from steward.models.users import User, UserRole
+
+    t = _tools()
+    tag = uuid.uuid4().hex[:8]
+    ids = {}
+
+    async def _seed():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"mm-{tag}", email=f"mm-{tag}@test", password_hash="x",
+                            role=UserRole.admin)
+                group = AnsibleGroup(name=f"mm-{tag}")
+                s.add_all([user, group])
+                await s.flush()
+                ids["user"], ids["group"] = user.id, group.id
+                for n in ("failed", "busy", "free"):
+                    host = Host(name=f"mm-{n}-{tag}", address="10.9.9.5")
+                    s.add(host)
+                    await s.flush()
+                    target = AnsibleTarget(name=f"mm-{n}-{tag}", address="10.9.9.5", host_id=host.id)
+                    target.groups = [group]
+                    s.add(target)
+                    await s.execute(text(
+                        "INSERT INTO host_agent_registrations (id, host_id, token_hash, "
+                        " token_created_at, last_seen_at, maint_helper, created_at, updated_at) "
+                        "VALUES (:id, :h, :hash, now(), now(), true, now(), now())"),
+                        {"id": str(uuid.uuid4()), "h": host.id,
+                         "hash": hashlib.sha256(uuid.uuid4().hex.encode()).hexdigest()})
+                    ids[n] = host.id
+                ids["plan"], ids["run"], ids["failed_job"] = (str(uuid.uuid4()) for _ in range(3))
+                await s.execute(text(
+                    "INSERT INTO host_maintenance_plans (id, name, group_id, weekday, at_time, "
+                    " time_zone, window_minutes, reboot_policy, enabled, created_by, created_at) "
+                    "VALUES (:id, :name, :g, 6, '03:00', 'UTC', 240, 'if_required', true, :u, now())"),
+                    {"id": ids["plan"], "name": f"plan-{tag}", "g": group.id, "u": user.id})
+                await s.execute(text(
+                    "INSERT INTO host_maintenance_runs (id, plan_id, state, host_ids, position, "
+                    " skipped, detail, manual, started_at, finished_at) "
+                    "VALUES (:id, :p, 'failed', :hosts, 1, :skipped, 'stopped: a host failed', "
+                    " false, now() - interval '2 hours', now() - interval '1 hour')"),
+                    {"id": ids["run"], "p": ids["plan"],
+                     "hosts": f'["{ids["failed"]}", "{ids["busy"]}"]',
+                     "skipped": f'[{{"host_id": "{ids["free"]}", "reason": "no agent"}}]'})
+                await s.execute(text(
+                    "INSERT INTO host_maintenance_jobs (id, host_id, state, state_at, reboot_policy, "
+                    " step_seen, packages, reboot_needed, detail, run_id, created_at, finished_at) "
+                    "VALUES (:id, :h, 'failed', now() - interval '1 hour', 'if_required', true, 3, "
+                    " true, 'update failed: apt-get dist-upgrade exited 100', :run, "
+                    " now() - interval '2 hours', now() - interval '1 hour')"),
+                    {"id": ids["failed_job"], "h": ids["failed"], "run": ids["run"]})
+                await s.execute(text(
+                    "INSERT INTO host_maintenance_jobs (id, host_id, state, state_at, reboot_policy, "
+                    " step_seen, created_by, created_at) "
+                    "VALUES (:id, :h, 'updating', now(), 'never', false, :u, now())"),
+                    {"id": str(uuid.uuid4()), "h": ids["busy"], "u": user.id})
+
+    async def _go():
+        await _seed()
+        _context._app_ctx.set(app)
+        _context._caller_ctx.set(Caller(user_id=ids["user"], username=f"mm-{tag}",
+                                        role=UserRole.admin, token_id=9))
+
+        plans = {p["plan_id"]: p for p in (await t.list_maintenance_plans())["plans"]}
+        plan = plans[ids["plan"]]
+        assert plan["group"] == f"mm-{tag}" and plan["schedule"].startswith("Sundays at 03:00")
+        assert plan["next_slot_at"] and plan["enabled"] is True
+        run = plan["latest_run"]
+        assert run["state"] == "failed" and run["hosts_done"] == 1
+        assert run["hosts"] == [f"mm-failed-{tag}", f"mm-busy-{tag}"]
+        assert run["skipped"] == [{"host": f"mm-free-{tag}", "reason": "no agent"}]
+
+        failed = (await t.get_maintenance_job(host=f"mm-failed-{tag}"))["jobs"]
+        assert [j["state"] for j in failed] == ["failed"]
+        assert failed[0]["detail"].startswith("update failed") and failed[0]["started_by"] == "maintenance plan"
+        by_id = (await t.get_maintenance_job(job=ids["failed_job"]))["jobs"][0]
+        assert by_id["host"] == f"mm-failed-{tag}"
+        with pytest.raises(ValueError, match="either `host`"):
+            await t.get_maintenance_job()
+
+        with pytest.raises(ValueError, match="already running.*Nothing was started"):
+            await t.run_maintenance(f"mm-busy-{tag}")
+        started = await t.run_maintenance(f"mm-free-{tag}", reboot="never")
+        assert started["state"] == "queued" and started["reboot_policy"] == "never"
+        [queued] = (await t.get_maintenance_job(host=ids["free"]))["jobs"]
+        assert queued["job_id"] == started["job_id"] and queued["started_by"] == f"mm-{tag}"
+
+        async with app.db_sessionmaker() as s:
+            rows = list((await s.execute(select(AuditEvent).where(
+                AuditEvent.user_id == ids["user"],
+                AuditEvent.action == "host_agent.maintenance"))).scalars())
+            findings = await t.mcp_overview(s)
+        assert len(rows) == 1 and rows[0].username == f"mm-{tag} via MCP"
+        mine = [f for f in findings if f["kind"] == "maintenance_failed"
+                and f["ids"]["host_id"] == ids["failed"]]
+        assert len(mine) == 1 and "apt-get dist-upgrade" in mine[0]["summary"]
+        assert not [f for f in findings if f["kind"] == "maintenance_failed"
+                    and f["ids"]["host_id"] in (ids["busy"], ids["free"])]
+
+    asyncio.run(_go())
+
+
+def test_the_host_agent_tools_are_listed_whatever_its_stored_switch_says(app):
+    """The host agent is always on (operator, 2026-10-15): its tools stay
+    listed even with a stored disable."""
+    from steward.core.settings import to_plugins_cfg
+    app.config["PLUGINS"] = to_plugins_cfg({"plugin.host_agent": {"enabled": False}})
+    mcp = app.mcp_server
+    assert mcp.plugin_enabled("host_agent")
+    for name in ("list_agents", "list_maintenance_plans", "get_maintenance_job", "run_maintenance"):
+        assert mcp.tool_plugins.get(name) == "host_agent", name

tests/plugins/host_agent/test_mcp_tools.py

@@ -18,5 +18,28 @@ def test_agent_state():
 def test_plugin_exposes_the_mcp_hooks():
     reg = MagicMock()
     agent_plugin.register_mcp_tools(reg)
-    assert [c.args[0].__name__ for c in reg.call_args_list] == ["list_agents"]
+    assert [c.args[0].__name__ for c in reg.call_args_list] == [
+        "list_agents", "list_maintenance_plans", "get_maintenance_job", "run_maintenance"]
+    roles = {c.args[0].__name__: c.kwargs["role"] for c in reg.call_args_list}
+    assert roles["run_maintenance"].value == "admin"           # as Update now on the page
+    assert roles["get_maintenance_job"].value == "viewer"
     assert callable(agent_plugin.mcp_overview)
+
+
+def test_a_job_row_says_what_it_did_and_when_it_gives_up():
+    from plugins.host_agent.mcp_tools import _job_row
+    job = NS(id="j1", host_id="h1", state="rebooting", state_at=NOW - timedelta(minutes=5),
+             reboot_policy="if_required", packages=12, reboot_needed=True, back_at=None,
+             node_id="n1", drain_host_id="h2", detail=None, created_by=None, run_id="r1",
+             created_at=NOW - timedelta(minutes=20), finished_at=None)
+    row = _job_row(job, {"h1": "web-1", "h2": "mgr-1"}, NOW)
+    assert (row["host"], row["drained_by"], row["label"], row["active"]) == (
+        "web-1", "mgr-1", "Rebooting", True)
+    assert row["started_by"] == "maintenance plan" and row["state_age_s"] == 300
+    assert row["gives_up_at"] == (NOW + timedelta(minutes=25)).isoformat().replace("+00:00", "Z")
+
+    done = NS(**{**job.__dict__, "state": "done", "back_at": NOW, "finished_at": NOW,
+                 "created_by": "u1", "run_id": None})
+    row = _job_row(done, {"h1": "web-1", "u1": "alice"}, NOW)
+    assert row["summary"] == "Done: 12 packages updated, rebooted"
+    assert row["started_by"] == "alice" and row["gives_up_at"] is None and not row["active"]