bvandeusenandClaude Opus 5.5
8e2f17d198
18032a4 re-checked a "not back in service" job every minute with no end,
so any later swarm wobble on the host (radarr's failed task during the
downloads move, say) flipped maintenance_failed back to 1: five false
firings and ten emails for Docker02 overnight on 10-19. And the overview's
finding never learned the rule at all, so it still said Docker02's
maintenance failed three days on.
The job now keeps came_back_at, set the first time its host is found back.
maintenance.still_failed(job) is the one answer: the figure, the overview,
the cadence page's run rows and the host panel all ask it. A failed update
or a timed-out job still flags its host until a job succeeds.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
173 lines
9.1 KiB
Python
173 lines
9.1 KiB
Python
# plugins/host_agent/models.py
|
|
from __future__ import annotations
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from sqlalchemy import JSON, Boolean, Column, String, DateTime, Float, ForeignKey, Integer, Text
|
|
from steward.models.base import Base
|
|
|
|
|
|
def _uuid() -> str:
|
|
return str(uuid.uuid4())
|
|
|
|
|
|
def _utcnow() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
class HostAgentRegistration(Base):
|
|
__tablename__ = "host_agent_registrations"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
|
|
unique=True, nullable=False)
|
|
token_hash = Column(String(64), nullable=False, unique=True, index=True)
|
|
token_created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
agent_version = Column(String(32), nullable=True)
|
|
kernel = Column(String(128), nullable=True)
|
|
distro = Column(String(128), nullable=True)
|
|
arch = Column(String(32), nullable=True)
|
|
# Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix).
|
|
host_ip = Column(String(45), nullable=True)
|
|
last_seen_at = Column(DateTime(timezone=True), nullable=True)
|
|
# How far this host's clock is from Steward's, in seconds, positive when
|
|
# the host is ahead, measured from the agent's sent_at on the newest post;
|
|
# and when that was. None until an agent that stamps its posts reports.
|
|
clock_offset_s = Column(Float, nullable=True)
|
|
clock_checked_at = Column(DateTime(timezone=True), nullable=True)
|
|
# What the package manager has waiting, from the agent's hourly check
|
|
# (agent ≥ 1.16.0, #5765). updates_security is None when the manager
|
|
# can't say; updates_error is why the last check failed.
|
|
updates_manager = Column(String(8), nullable=True)
|
|
updates_pending = Column(Integer, nullable=True)
|
|
updates_security = Column(Integer, nullable=True)
|
|
updates_checked_at = Column(DateTime(timezone=True), nullable=True)
|
|
updates_error = Column(String(200), nullable=True)
|
|
# Whether the host needs a reboot (None: it can't say), since when, and
|
|
# the packages that asked for it, comma-separated.
|
|
reboot_required = Column(Boolean, nullable=True)
|
|
reboot_required_since = Column(DateTime(timezone=True), nullable=True)
|
|
reboot_packages = Column(Text, nullable=True)
|
|
# Whether the host has the root maintenance helper (steward-maint.path and
|
|
# its request directory) that updates and reboots go through (#5766); None
|
|
# until an agent that says reports.
|
|
maint_helper = Column(Boolean, nullable=True)
|
|
# The host's newest drive-health report, as the root helper read it with
|
|
# smartctl (agent ≥ 1.20.0, #5789): {"checked_at", "smartctl", "drives"} or
|
|
# {"error"}. drives.py judges it; the figures are also metrics.
|
|
drive_health = Column(JSON, nullable=True)
|
|
drive_health_at = Column(DateTime(timezone=True), nullable=True)
|
|
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
updated_at = Column(DateTime(timezone=True), nullable=False,
|
|
default=_utcnow, onupdate=_utcnow)
|
|
|
|
|
|
class HostDowntime(Base):
|
|
"""A span a host's agent was away, and what the agent said it was (#5756).
|
|
|
|
kind is "reboot" (the host went down and came back), "agent_stop" (the
|
|
agent stopped; the host stayed up), "unclean" (the host came back from a
|
|
boot nobody announced: power loss or a crash) or "lost_contact" (the agent
|
|
ran but couldn't reach Steward). ended_at is None while it is going on.
|
|
"""
|
|
__tablename__ = "host_downtime"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
|
|
nullable=False, index=True)
|
|
kind = Column(String(16), nullable=False)
|
|
started_at = Column(DateTime(timezone=True), nullable=False)
|
|
ended_at = Column(DateTime(timezone=True), nullable=True)
|
|
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
|
|
|
|
class HostMaintenanceJob(Base):
|
|
"""One host's update, and its reboot when the update needs one (#5767).
|
|
|
|
Steward keeps the job here and the host's agent carries out each step
|
|
through its root helper; maintenance.py has the states and transitions.
|
|
step_id is the helper request id of the current step, and step_seen says a
|
|
result for it has arrived, so it stops being handed out. node_id and
|
|
drain_host_id are set when the host is a swarm node: the node to drain and
|
|
the manager whose agent drains it.
|
|
"""
|
|
__tablename__ = "host_maintenance_jobs"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
|
|
nullable=False, index=True)
|
|
state = Column(String(16), nullable=False, default="queued", index=True)
|
|
state_at = Column(DateTime(timezone=True), nullable=True)
|
|
reboot_policy = Column(String(16), nullable=False, default="if_required")
|
|
step_id = Column(String(64), nullable=True, index=True)
|
|
step_seen = Column(Boolean, nullable=False, default=False)
|
|
node_id = Column(String(64), nullable=True)
|
|
drain_host_id = Column(String(36), ForeignKey("hosts.id", ondelete="SET NULL"), nullable=True)
|
|
packages = Column(Integer, nullable=True)
|
|
reboot_needed = Column(Boolean, nullable=True)
|
|
back_at = Column(DateTime(timezone=True), nullable=True)
|
|
detail = Column(Text, nullable=True)
|
|
window_id = Column(String(36), ForeignKey("maintenance_windows.id", ondelete="SET NULL"),
|
|
nullable=True)
|
|
created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
|
|
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
finished_at = Column(DateTime(timezone=True), nullable=True)
|
|
# The cadence run this job belongs to (#5768); None for Update now.
|
|
run_id = Column(String(36), ForeignKey("host_maintenance_runs.id", ondelete="SET NULL"),
|
|
nullable=True, index=True)
|
|
# What the check said the update would change (#6098): whether it
|
|
# upgrades Docker, and {pending, docker_packages, containers} with the
|
|
# containers then running outside the swarm. drained_at is when the job's
|
|
# drain finished, before the update or before the reboot.
|
|
docker_update = Column(Boolean, nullable=True)
|
|
checked = Column(JSON, nullable=True)
|
|
drained_at = Column(DateTime(timezone=True), nullable=True)
|
|
# A job failed for not coming back into service in time: when its host
|
|
# was first found back (#6175). From then on the job no longer flags it.
|
|
came_back_at = Column(DateTime(timezone=True), nullable=True)
|
|
|
|
|
|
class HostMaintenancePlan(Base):
|
|
"""A group's maintenance cadence (#5768): every week on `weekday` at
|
|
`at_time` in `time_zone`, its hosts are updated one at a time, each
|
|
rebooting per `reboot_policy`. No host is started after `window_minutes`
|
|
from the run's start. last_slot_at is the last scheduled slot that fired;
|
|
Run now leaves it alone, so the schedule keeps its time."""
|
|
__tablename__ = "host_maintenance_plans"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
name = Column(String(128), nullable=False)
|
|
group_id = Column(String(36), ForeignKey("ansible_groups.id", ondelete="CASCADE"),
|
|
nullable=False)
|
|
weekday = Column(Integer, nullable=False) # 0 = Monday
|
|
at_time = Column(String(5), nullable=False) # "HH:MM"
|
|
time_zone = Column(String(64), nullable=False, default="UTC")
|
|
window_minutes = Column(Integer, nullable=False, default=240)
|
|
reboot_policy = Column(String(16), nullable=False, default="if_required")
|
|
enabled = Column(Boolean, nullable=False, default=True)
|
|
last_slot_at = Column(DateTime(timezone=True), nullable=True)
|
|
created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
|
|
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
|
|
|
|
class HostMaintenanceRun(Base):
|
|
"""One run of a plan: its hosts in order, one at a time (#5768).
|
|
|
|
host_ids is the order fixed at the start (JSON list); position is the host
|
|
being worked on; skipped is [{"host_id", "reason"}] for hosts the run
|
|
passed over. settle_since is when the current host's job finished and the
|
|
run began waiting for the host to be back in service."""
|
|
__tablename__ = "host_maintenance_runs"
|
|
|
|
id = Column(String(36), primary_key=True, default=_uuid)
|
|
plan_id = Column(String(36), ForeignKey("host_maintenance_plans.id", ondelete="CASCADE"),
|
|
nullable=False, index=True)
|
|
state = Column(String(16), nullable=False, default="running", index=True)
|
|
host_ids = Column(Text, nullable=False, default="[]")
|
|
position = Column(Integer, nullable=False, default=0)
|
|
skipped = Column(Text, nullable=False, default="[]")
|
|
settle_since = Column(DateTime(timezone=True), nullable=True)
|
|
detail = Column(Text, nullable=True)
|
|
manual = Column(Boolean, nullable=False, default=False)
|
|
started_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
finished_at = Column(DateTime(timezone=True), nullable=True)
|