bvandeusenandClaude Opus 5.5
22b0be2b6c
#5767 "Steward hands a host a maintenance job and follows it to the end, through its own restarts", milestone 514. A host_maintenance_jobs row per job: queued → updating → [draining →] rebooting → [undraining →] done, or failed / timed_out. Steward's answer to an ingest post carries the step that host's agent should carry out now; the agent (1.18.0) leaves each step id for the root helper once, and the helper refuses an id it already ran, so repeats are harmless. Results come back as the #5766 maintenance events, matched by step id; the reboot completes when the host's `started` event has a boot_at after the reboot was handed out. The job reboots only when the update says one is needed (or never, if asked). A swarm node is drained first by itself if it's a manager or by a manager with the helper, and made active after; with no manager to drain it the job is refused. A minute tick times out a job past its state's deadline. A maintenance window over host_agent/<host> lasts the rest of the job. Everything is on the row, so a Steward restart, or the reboot of the host Steward runs on, resumes from the next post. Ingest runs it in its own savepoint so a job going wrong never costs the samples. The Agent panel leads with a running job and says what a finished one did; Manage agent has Update now (with "never reboot"), or says why a job can't start. Integration test: hand off, Steward restarts, the post-boot events close the job. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
109 lines
5.4 KiB
Python
109 lines
5.4 KiB
Python
# plugins/host_agent/models.py
|
|
from __future__ import annotations
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from sqlalchemy import Boolean, Column, String, DateTime, Float, ForeignKey, Integer, Text
|
|
from steward.models.base import Base
|
|
|
|
|
|
def _uuid() -> str:
|
|
return str(uuid.uuid4())
|
|
|
|
|
|
def _utcnow() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
class HostAgentRegistration(Base):
|
|
__tablename__ = "host_agent_registrations"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
|
|
unique=True, nullable=False)
|
|
token_hash = Column(String(64), nullable=False, unique=True, index=True)
|
|
token_created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
agent_version = Column(String(32), nullable=True)
|
|
kernel = Column(String(128), nullable=True)
|
|
distro = Column(String(128), nullable=True)
|
|
arch = Column(String(32), nullable=True)
|
|
# Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix).
|
|
host_ip = Column(String(45), nullable=True)
|
|
last_seen_at = Column(DateTime(timezone=True), nullable=True)
|
|
# How far this host's clock is from Steward's, in seconds, positive when
|
|
# the host is ahead, measured from the agent's sent_at on the newest post;
|
|
# and when that was. None until an agent that stamps its posts reports.
|
|
clock_offset_s = Column(Float, nullable=True)
|
|
clock_checked_at = Column(DateTime(timezone=True), nullable=True)
|
|
# What the package manager has waiting, from the agent's hourly check
|
|
# (agent ≥ 1.16.0, #5765). updates_security is None when the manager
|
|
# can't say; updates_error is why the last check failed.
|
|
updates_manager = Column(String(8), nullable=True)
|
|
updates_pending = Column(Integer, nullable=True)
|
|
updates_security = Column(Integer, nullable=True)
|
|
updates_checked_at = Column(DateTime(timezone=True), nullable=True)
|
|
updates_error = Column(String(200), nullable=True)
|
|
# Whether the host needs a reboot (None: it can't say), since when, and
|
|
# the packages that asked for it, comma-separated.
|
|
reboot_required = Column(Boolean, nullable=True)
|
|
reboot_required_since = Column(DateTime(timezone=True), nullable=True)
|
|
reboot_packages = Column(Text, nullable=True)
|
|
# Whether the host has the root maintenance helper (steward-maint.path and
|
|
# its request directory) that updates and reboots go through (#5766); None
|
|
# until an agent that says reports.
|
|
maint_helper = Column(Boolean, nullable=True)
|
|
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
updated_at = Column(DateTime(timezone=True), nullable=False,
|
|
default=_utcnow, onupdate=_utcnow)
|
|
|
|
|
|
class HostDowntime(Base):
|
|
"""A span a host's agent was away, and what the agent said it was (#5756).
|
|
|
|
kind is "reboot" (the host went down and came back), "agent_stop" (the
|
|
agent stopped; the host stayed up), "unclean" (the host came back from a
|
|
boot nobody announced: power loss or a crash) or "lost_contact" (the agent
|
|
ran but couldn't reach Steward). ended_at is None while it is going on.
|
|
"""
|
|
__tablename__ = "host_downtime"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
|
|
nullable=False, index=True)
|
|
kind = Column(String(16), nullable=False)
|
|
started_at = Column(DateTime(timezone=True), nullable=False)
|
|
ended_at = Column(DateTime(timezone=True), nullable=True)
|
|
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
|
|
|
|
class HostMaintenanceJob(Base):
|
|
"""One host's update, and its reboot when the update needs one (#5767).
|
|
|
|
Steward keeps the job here and the host's agent carries out each step
|
|
through its root helper; maintenance.py has the states and transitions.
|
|
step_id is the helper request id of the current step, and step_seen says a
|
|
result for it has arrived, so it stops being handed out. node_id and
|
|
drain_host_id are set when the host is a swarm node: the node to drain and
|
|
the manager whose agent drains it.
|
|
"""
|
|
__tablename__ = "host_maintenance_jobs"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
|
|
nullable=False, index=True)
|
|
state = Column(String(16), nullable=False, default="queued", index=True)
|
|
state_at = Column(DateTime(timezone=True), nullable=True)
|
|
reboot_policy = Column(String(16), nullable=False, default="if_required")
|
|
step_id = Column(String(64), nullable=True, index=True)
|
|
step_seen = Column(Boolean, nullable=False, default=False)
|
|
node_id = Column(String(64), nullable=True)
|
|
drain_host_id = Column(String(36), ForeignKey("hosts.id", ondelete="SET NULL"), nullable=True)
|
|
packages = Column(Integer, nullable=True)
|
|
reboot_needed = Column(Boolean, nullable=True)
|
|
back_at = Column(DateTime(timezone=True), nullable=True)
|
|
detail = Column(Text, nullable=True)
|
|
window_id = Column(String(36), ForeignKey("maintenance_windows.id", ondelete="SET NULL"),
|
|
nullable=True)
|
|
created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
|
|
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
finished_at = Column(DateTime(timezone=True), nullable=True)
|