bvandeusenandClaude Opus 5.5
d102fc3333
On 2026-10-25 the 09:00 job on VDNT-Docker02 drained the node, which ran Traefik and Steward. The agent's result reaches Steward through Traefik, and Traefik took 11.5 minutes to start on Docker03 (its image had to be pulled). The job gave up at the 10-minute drain deadline and finished timed_out where it stood, so the node stayed drained for hours. - A job that fails or times out while its node may be drained (the drain was handed out, or went through and wasn't undone) goes to undraining first. The outcome is kept in the new column ends_as, and the job finishes as that once the node is active again. If the undrain goes wrong too, detail says both. - The drain deadline is 30 minutes instead of 10, so a result held up while the proxy moves still lands. - Migration host_agent_012_ends_as. get_maintenance_job returns ends_as. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
176 lines
9.3 KiB
Python
176 lines
9.3 KiB
Python
# plugins/host_agent/models.py
|
|
from __future__ import annotations
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from sqlalchemy import JSON, Boolean, Column, String, DateTime, Float, ForeignKey, Integer, Text
|
|
from steward.models.base import Base
|
|
|
|
|
|
def _uuid() -> str:
|
|
return str(uuid.uuid4())
|
|
|
|
|
|
def _utcnow() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
class HostAgentRegistration(Base):
|
|
__tablename__ = "host_agent_registrations"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
|
|
unique=True, nullable=False)
|
|
token_hash = Column(String(64), nullable=False, unique=True, index=True)
|
|
token_created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
agent_version = Column(String(32), nullable=True)
|
|
kernel = Column(String(128), nullable=True)
|
|
distro = Column(String(128), nullable=True)
|
|
arch = Column(String(32), nullable=True)
|
|
# Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix).
|
|
host_ip = Column(String(45), nullable=True)
|
|
last_seen_at = Column(DateTime(timezone=True), nullable=True)
|
|
# How far this host's clock is from Steward's, in seconds, positive when
|
|
# the host is ahead, measured from the agent's sent_at on the newest post;
|
|
# and when that was. None until an agent that stamps its posts reports.
|
|
clock_offset_s = Column(Float, nullable=True)
|
|
clock_checked_at = Column(DateTime(timezone=True), nullable=True)
|
|
# What the package manager has waiting, from the agent's hourly check
|
|
# (agent ≥ 1.16.0, #5765). updates_security is None when the manager
|
|
# can't say; updates_error is why the last check failed.
|
|
updates_manager = Column(String(8), nullable=True)
|
|
updates_pending = Column(Integer, nullable=True)
|
|
updates_security = Column(Integer, nullable=True)
|
|
updates_checked_at = Column(DateTime(timezone=True), nullable=True)
|
|
updates_error = Column(String(200), nullable=True)
|
|
# Whether the host needs a reboot (None: it can't say), since when, and
|
|
# the packages that asked for it, comma-separated.
|
|
reboot_required = Column(Boolean, nullable=True)
|
|
reboot_required_since = Column(DateTime(timezone=True), nullable=True)
|
|
reboot_packages = Column(Text, nullable=True)
|
|
# Whether the host has the root maintenance helper (steward-maint.path and
|
|
# its request directory) that updates and reboots go through (#5766); None
|
|
# until an agent that says reports.
|
|
maint_helper = Column(Boolean, nullable=True)
|
|
# The host's newest drive-health report, as the root helper read it with
|
|
# smartctl (agent ≥ 1.20.0, #5789): {"checked_at", "smartctl", "drives"} or
|
|
# {"error"}. drives.py judges it; the figures are also metrics.
|
|
drive_health = Column(JSON, nullable=True)
|
|
drive_health_at = Column(DateTime(timezone=True), nullable=True)
|
|
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
updated_at = Column(DateTime(timezone=True), nullable=False,
|
|
default=_utcnow, onupdate=_utcnow)
|
|
|
|
|
|
class HostDowntime(Base):
|
|
"""A span a host's agent was away, and what the agent said it was (#5756).
|
|
|
|
kind is "reboot" (the host went down and came back), "agent_stop" (the
|
|
agent stopped; the host stayed up), "unclean" (the host came back from a
|
|
boot nobody announced: power loss or a crash) or "lost_contact" (the agent
|
|
ran but couldn't reach Steward). ended_at is None while it is going on.
|
|
"""
|
|
__tablename__ = "host_downtime"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
|
|
nullable=False, index=True)
|
|
kind = Column(String(16), nullable=False)
|
|
started_at = Column(DateTime(timezone=True), nullable=False)
|
|
ended_at = Column(DateTime(timezone=True), nullable=True)
|
|
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
|
|
|
|
class HostMaintenanceJob(Base):
|
|
"""One host's update, and its reboot when the update needs one (#5767).
|
|
|
|
Steward keeps the job here and the host's agent carries out each step
|
|
through its root helper; maintenance.py has the states and transitions.
|
|
step_id is the helper request id of the current step, and step_seen says a
|
|
result for it has arrived, so it stops being handed out. node_id and
|
|
drain_host_id are set when the host is a swarm node: the node to drain and
|
|
the manager whose agent drains it.
|
|
"""
|
|
__tablename__ = "host_maintenance_jobs"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
|
|
nullable=False, index=True)
|
|
state = Column(String(16), nullable=False, default="queued", index=True)
|
|
state_at = Column(DateTime(timezone=True), nullable=True)
|
|
reboot_policy = Column(String(16), nullable=False, default="if_required")
|
|
step_id = Column(String(64), nullable=True, index=True)
|
|
step_seen = Column(Boolean, nullable=False, default=False)
|
|
node_id = Column(String(64), nullable=True)
|
|
drain_host_id = Column(String(36), ForeignKey("hosts.id", ondelete="SET NULL"), nullable=True)
|
|
packages = Column(Integer, nullable=True)
|
|
reboot_needed = Column(Boolean, nullable=True)
|
|
back_at = Column(DateTime(timezone=True), nullable=True)
|
|
detail = Column(Text, nullable=True)
|
|
window_id = Column(String(36), ForeignKey("maintenance_windows.id", ondelete="SET NULL"),
|
|
nullable=True)
|
|
created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
|
|
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
finished_at = Column(DateTime(timezone=True), nullable=True)
|
|
# The cadence run this job belongs to (#5768); None for Update now.
|
|
run_id = Column(String(36), ForeignKey("host_maintenance_runs.id", ondelete="SET NULL"),
|
|
nullable=True, index=True)
|
|
# What the check said the update would change (#6098): whether it
|
|
# upgrades Docker, and {pending, docker_packages, containers} with the
|
|
# containers then running outside the swarm. drained_at is when the job's
|
|
# drain finished, before the update or before the reboot.
|
|
docker_update = Column(Boolean, nullable=True)
|
|
checked = Column(JSON, nullable=True)
|
|
drained_at = Column(DateTime(timezone=True), nullable=True)
|
|
# A job failed for not coming back into service in time: when its host
|
|
# was first found back (#6175). From then on the job no longer flags it.
|
|
came_back_at = Column(DateTime(timezone=True), nullable=True)
|
|
# A job given up (failed or timed_out) while its node may be drained makes
|
|
# the node active again first; ends_as is what it finishes as after that.
|
|
ends_as = Column(String(16), nullable=True)
|
|
|
|
|
|
class HostMaintenancePlan(Base):
|
|
"""A group's maintenance cadence (#5768): every week on `weekday` at
|
|
`at_time` in `time_zone`, its hosts are updated one at a time, each
|
|
rebooting per `reboot_policy`. No host is started after `window_minutes`
|
|
from the run's start. last_slot_at is the last scheduled slot that fired;
|
|
Run now leaves it alone, so the schedule keeps its time."""
|
|
__tablename__ = "host_maintenance_plans"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
name = Column(String(128), nullable=False)
|
|
group_id = Column(String(36), ForeignKey("ansible_groups.id", ondelete="CASCADE"),
|
|
nullable=False)
|
|
weekday = Column(Integer, nullable=False) # 0 = Monday
|
|
at_time = Column(String(5), nullable=False) # "HH:MM"
|
|
time_zone = Column(String(64), nullable=False, default="UTC")
|
|
window_minutes = Column(Integer, nullable=False, default=240)
|
|
reboot_policy = Column(String(16), nullable=False, default="if_required")
|
|
enabled = Column(Boolean, nullable=False, default=True)
|
|
last_slot_at = Column(DateTime(timezone=True), nullable=True)
|
|
created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
|
|
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
|
|
|
|
class HostMaintenanceRun(Base):
|
|
"""One run of a plan: its hosts in order, one at a time (#5768).
|
|
|
|
host_ids is the order fixed at the start (JSON list); position is the host
|
|
being worked on; skipped is [{"host_id", "reason"}] for hosts the run
|
|
passed over. settle_since is when the current host's job finished and the
|
|
run began waiting for the host to be back in service."""
|
|
__tablename__ = "host_maintenance_runs"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
plan_id = Column(String(36), ForeignKey("host_maintenance_plans.id", ondelete="CASCADE"),
|
|
nullable=False, index=True)
|
|
state = Column(String(16), nullable=False, default="running", index=True)
|
|
host_ids = Column(Text, nullable=False, default="[]")
|
|
position = Column(Integer, nullable=False, default=0)
|
|
skipped = Column(Text, nullable=False, default="[]")
|
|
settle_since = Column(DateTime(timezone=True), nullable=True)
|
|
detail = Column(Text, nullable=True)
|
|
manual = Column(Boolean, nullable=False, default=False)
|
|
started_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
finished_at = Column(DateTime(timezone=True), nullable=True)
|