bvandeusenandClaude Opus 5.5
22b0be2b6c
#5767 "Steward hands a host a maintenance job and follows it to the end, through its own restarts", milestone 514. A host_maintenance_jobs row per job: queued → updating → [draining →] rebooting → [undraining →] done, or failed / timed_out. Steward's answer to an ingest post carries the step that host's agent should carry out now; the agent (1.18.0) leaves each step id for the root helper once, and the helper refuses an id it already ran, so repeats are harmless. Results come back as the #5766 maintenance events, matched by step id; the reboot completes when the host's `started` event has a boot_at after the reboot was handed out. The job reboots only when the update says one is needed (or never, if asked). A swarm node is drained first by itself if it's a manager or by a manager with the helper, and made active after; with no manager to drain it the job is refused. A minute tick times out a job past its state's deadline. A maintenance window over host_agent/<host> lasts the rest of the job. Everything is on the row, so a Steward restart, or the reboot of the host Steward runs on, resumes from the next post. Ingest runs it in its own savepoint so a job going wrong never costs the samples. The Agent panel leads with a running job and says what a finished one did; Manage agent has Update now (with "never reboot"), or says why a job can't start. Integration test: hand off, Steward restarts, the post-boot events close the job. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
119 lines
5.2 KiB
Python
119 lines
5.2 KiB
Python
"""Integration (#5767): Steward hands a host a maintenance job and follows it
|
|
to the end, through a restart of its own, from the agent's posts alone.
|
|
|
|
Rows are seeded and read with plain SQL: importing the plugin's models again
|
|
after the app loaded the plugin raises "Table already defined" (see
|
|
test_host_agent_alerts)."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import os
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
def _app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
async def _seed(app):
|
|
from sqlalchemy import text
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import User
|
|
tag = uuid.uuid4().hex[:8]
|
|
token = f"tok-{tag}-{uuid.uuid4().hex}"
|
|
job_id = str(uuid.uuid4())
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"mj-{tag}", email=f"mj-{tag}@test", password_hash="x")
|
|
host = Host(name=f"maint-{tag}", address="10.9.9.7")
|
|
s.add_all([user, host])
|
|
await s.flush()
|
|
await s.execute(text(
|
|
"INSERT INTO host_agent_registrations "
|
|
"(id, host_id, token_hash, token_created_at, last_seen_at, maint_helper, "
|
|
" created_at, updated_at) "
|
|
"VALUES (:id, :host, :hash, now(), now(), true, now(), now())"),
|
|
{"id": str(uuid.uuid4()), "host": host.id,
|
|
"hash": hashlib.sha256(token.encode()).hexdigest()})
|
|
await s.execute(text(
|
|
"INSERT INTO host_maintenance_jobs "
|
|
"(id, host_id, state, state_at, reboot_policy, step_seen, created_by, created_at) "
|
|
"VALUES (:id, :host, 'queued', now(), 'if_required', false, :user, now())"),
|
|
{"id": job_id, "host": host.id, "user": user.id})
|
|
host_id = host.id
|
|
return host_id, token, job_id
|
|
|
|
|
|
async def _post(app, token, events=(), sample=True):
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
resp = await app.test_client().post(
|
|
"/plugins/host_agent/ingest",
|
|
json={"samples": [{"ts": now, "cpu_pct": 2.0}] if sample else [],
|
|
"events": list(events)},
|
|
headers={"Authorization": f"Bearer {token}"})
|
|
assert resp.status_code == 200, await resp.get_data(as_text=True)
|
|
return (await resp.get_json()).get("maintenance", [])
|
|
|
|
|
|
async def _job(app, job_id):
|
|
from sqlalchemy import text
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
return (await s.execute(text(
|
|
"SELECT j.state, j.packages, j.reboot_needed, j.back_at, w.end_at "
|
|
"FROM host_maintenance_jobs j LEFT JOIN maintenance_windows w ON w.id = j.window_id "
|
|
"WHERE j.id = :id"), {"id": job_id})).one()
|
|
|
|
|
|
def test_a_job_is_followed_through_a_steward_restart_and_the_host_reboot():
|
|
app = _app()
|
|
|
|
async def _first_half():
|
|
host_id, token, job_id = await _seed(app)
|
|
# The agent's next post starts the job: update first, under a window.
|
|
assert await _post(app, token) == [{"id": f"{job_id}-update", "action": "update"}]
|
|
state, *_, window_end = await _job(app, job_id)
|
|
assert state == "updating" and window_end > datetime.now(timezone.utc)
|
|
# The helper took it: the step is no longer handed out.
|
|
running = {"kind": "maintenance", "id": f"{job_id}-update", "action": "update",
|
|
"state": "running"}
|
|
assert await _post(app, token, [running]) == []
|
|
return token, job_id
|
|
|
|
token, job_id = asyncio.run(_first_half())
|
|
|
|
# Steward restarts: a new process, nothing in memory but the database.
|
|
app = _app()
|
|
|
|
async def _second_half():
|
|
done = {"kind": "maintenance", "id": f"{job_id}-update", "action": "update",
|
|
"state": "done", "ok": True, "packages": 12, "reboot_required": True}
|
|
assert await _post(app, token, [done]) == [{"id": f"{job_id}-reboot", "action": "reboot"}]
|
|
assert (await _job(app, job_id))[:3] == ("rebooting", 12, True)
|
|
# The agent restarting on a host that stayed up doesn't count as the reboot.
|
|
old_boot = datetime.now(timezone.utc) - timedelta(days=3)
|
|
await _post(app, token, [{"kind": "started", "at": datetime.now(timezone.utc).isoformat(),
|
|
"boot_at": old_boot.isoformat()}])
|
|
assert (await _job(app, job_id))[0] == "rebooting"
|
|
# After the reboot: the helper's result and the agent's boot, in one post.
|
|
boot = datetime.now(timezone.utc)
|
|
rebooted = {"kind": "maintenance", "id": f"{job_id}-reboot", "action": "reboot",
|
|
"state": "rebooting", "ok": True}
|
|
started = {"kind": "started", "at": (boot + timedelta(seconds=20)).isoformat(),
|
|
"boot_at": boot.isoformat()}
|
|
assert await _post(app, token, [rebooted, started]) == []
|
|
state, packages, reboot_needed, back_at, window_end = await _job(app, job_id)
|
|
assert state == "done" and back_at is not None
|
|
assert window_end <= datetime.now(timezone.utc)
|
|
|
|
asyncio.run(_second_half())
|