fix(maintenance): check first, drain before a Docker upgrade, wait 5 min after a drain, see containers back (#6098)
On 2026-10-18 an update on VDNT-Docker02 restarted the Docker daemon six seconds in, before the drain that came only with the reboot, and every container on the node stopped. Docker's own packages restart the daemon from their install scripts; needrestart's list-only mode doesn't stop that. - The helper gains a `check` action (agent 1.24.0): a dry run of the update for each package manager, whether Docker is among it, and the running containers outside the swarm with their restart policies. - A swarm node is drained before an update that upgrades Docker, else only before a reboot, and at most once a job. - After a drain the job settles 5 minutes before the update or reboot. - Once the work is over, the containers that ran outside the swarm must be seen running again within 10 minutes, or the job fails naming them. - An older agent gets no check and its job runs as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
# plugins/host_agent/migrations/versions/host_agent_010_maint_check.py
|
||||
"""host_agent: a maintenance job's check of what its update would change, and
|
||||
when it drained its node (#6098)
|
||||
|
||||
Revision ID: host_agent_010_maint_check
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "host_agent_010_maint_check"
|
||||
down_revision: Union[str, None] = "host_agent_009_drive_health"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("host_maintenance_jobs", sa.Column("docker_update", sa.Boolean(), nullable=True))
|
||||
op.add_column("host_maintenance_jobs", sa.Column("checked", sa.JSON(), nullable=True))
|
||||
op.add_column("host_maintenance_jobs",
|
||||
sa.Column("drained_at", sa.DateTime(timezone=True), nullable=True))
|
||||
# A job already past its drain when this lands still makes its node
|
||||
# active again: before this, every job that drained went on to undrain.
|
||||
op.execute("UPDATE host_maintenance_jobs SET drained_at = state_at "
|
||||
"WHERE state IN ('rebooting', 'undraining') "
|
||||
"AND node_id IS NOT NULL AND drain_host_id IS NOT NULL")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("host_maintenance_jobs", "drained_at")
|
||||
op.drop_column("host_maintenance_jobs", "checked")
|
||||
op.drop_column("host_maintenance_jobs", "docker_update")
|
||||
Reference in New Issue
Block a user