feat(homeassistant): a scheduled OS update reboots the machine and is done once the new version runs (#6755)
Home Assistant writes an OS update and leaves the reboot to someone, so a scheduled one never finished unattended. The schedule now asks for the reboot once, follows it through Home Assistant going away, and counts it done when the Supervisor reports a different version with nothing pending. A machine that doesn't come back or falls back to the old version fails after 20 minutes, which ends the slot and raises update_failed. Install on the page and through MCP still leave the reboot to the person. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,7 @@ import asyncio
|
||||
import json
|
||||
import ssl
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import httpx
|
||||
@@ -242,9 +243,9 @@ class _Socket:
|
||||
return await self.ask(type="supervisor/api", endpoint=endpoint, method="get")
|
||||
|
||||
|
||||
async def read_system(url: str, token: str, verify_tls: bool, connect=None) -> dict:
|
||||
"""What service.read_health needs, as Home Assistant answered it. `connect`
|
||||
stands in for the websocket in tests."""
|
||||
@asynccontextmanager
|
||||
async def _signed_in(url: str, token: str, verify_tls: bool, connect):
|
||||
"""A signed-in socket. `connect` stands in for the websocket in tests."""
|
||||
target = "ws" + url.rstrip("/").removeprefix("http") + "/api/websocket"
|
||||
options: dict = {"max_size": MAX_BYTES}
|
||||
if target.startswith("wss:") and not verify_tls:
|
||||
@@ -255,23 +256,37 @@ async def read_system(url: str, token: str, verify_tls: bool, connect=None) -> d
|
||||
async with (connect or ws_connect)(target, **options) as ws:
|
||||
socket = _Socket(ws)
|
||||
await socket.sign_in(token)
|
||||
found = {"config": await socket.ask(type="get_config"),
|
||||
"issues": (await socket.ask(type="repairs/list_issues") or {}).get("issues"),
|
||||
"resolution": await socket.supervisor("/resolution/info"),
|
||||
"host": await socket.supervisor("/host/info"), "titles": {}, "boots": {}}
|
||||
domains = sorted({i["domain"] for i in found["issues"] or []})
|
||||
if domains:
|
||||
titles = await socket.ask(type="frontend/get_translations", language="en",
|
||||
category="issues", integration=domains)
|
||||
found["titles"] = (titles or {}).get("resources") or {}
|
||||
found["addons"] = (await socket.supervisor("/addons") or {}).get("addons")
|
||||
# Whether one should be running is only on its own page, so a running one isn't asked.
|
||||
for addon in found["addons"] or []:
|
||||
if addon.get("state") != "started":
|
||||
info = await socket.supervisor(f"/addons/{addon['slug']}/info") or {}
|
||||
found["boots"][addon["slug"]] = info.get("boot")
|
||||
return found
|
||||
yield socket
|
||||
except TimeoutError:
|
||||
raise Unreachable(f"Home Assistant's system report didn't arrive within {SYSTEM_READ_SECONDS} s.") from None
|
||||
except (OSError, ValueError, KeyError, WebSocketException) as exc:
|
||||
raise Unreachable(f"Couldn't read Home Assistant's system report: {exc}") from None
|
||||
|
||||
|
||||
async def read_system(url: str, token: str, verify_tls: bool, connect=None) -> dict:
|
||||
"""What service.read_health needs, as Home Assistant answered it."""
|
||||
async with _signed_in(url, token, verify_tls, connect) as socket:
|
||||
found = {"config": await socket.ask(type="get_config"),
|
||||
"issues": (await socket.ask(type="repairs/list_issues") or {}).get("issues"),
|
||||
"resolution": await socket.supervisor("/resolution/info"),
|
||||
"host": await socket.supervisor("/host/info"), "titles": {}, "boots": {}}
|
||||
domains = sorted({i["domain"] for i in found["issues"] or []})
|
||||
if domains:
|
||||
titles = await socket.ask(type="frontend/get_translations", language="en",
|
||||
category="issues", integration=domains)
|
||||
found["titles"] = (titles or {}).get("resources") or {}
|
||||
found["addons"] = (await socket.supervisor("/addons") or {}).get("addons")
|
||||
# Whether one should be running is only on its own page, so a running one isn't asked.
|
||||
for addon in found["addons"] or []:
|
||||
if addon.get("state") != "started":
|
||||
info = await socket.supervisor(f"/addons/{addon['slug']}/info") or {}
|
||||
found["boots"][addon["slug"]] = info.get("boot")
|
||||
return found
|
||||
|
||||
|
||||
async def read_os(url: str, token: str, verify_tls: bool, connect=None) -> dict | None:
|
||||
"""The Supervisor's report on the OS: `version`, and `version_pending` while a
|
||||
written update waits for its reboot. None if it isn't answering yet, as just
|
||||
after a boot."""
|
||||
async with _signed_in(url, token, verify_tls, connect) as socket:
|
||||
return await socket.supervisor("/os/info")
|
||||
|
||||
@@ -105,9 +105,9 @@ async def set_home_assistant_schedule(enabled: bool, weekday: int = 5, at_time:
|
||||
|
||||
At each slot Steward installs add-ons, then Core, then the OS, one at a time,
|
||||
each after a confirmed backup, stopping at the first failure. A Core update
|
||||
restarts Home Assistant with nobody watching. An OS update takes effect after
|
||||
a reboot, which Steward leaves to a person, and `system.problems` shows it until
|
||||
then. The first slot is the next one from now. Admin only, and audited.
|
||||
restarts Home Assistant and an OS update reboots the machine, which Steward
|
||||
does itself here, with nobody watching. The first slot is the next one from
|
||||
now. Admin only, and audited.
|
||||
"""
|
||||
try:
|
||||
weekly.parse_at(at_time)
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Home Assistant: an update the schedule installs can reboot the machine and follow it
|
||||
|
||||
Revision ID: homeassistant_004_reboot
|
||||
Revises: homeassistant_003_health
|
||||
Create Date: 2026-11-02
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "homeassistant_004_reboot"
|
||||
down_revision: Union[str, None] = "homeassistant_003_health"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def _under_way(states: str) -> None:
|
||||
op.execute("DROP INDEX uq_homeassistant_update_under_way")
|
||||
op.execute("CREATE UNIQUE INDEX uq_homeassistant_update_under_way ON homeassistant_updates ((true)) "
|
||||
f"WHERE state IN ({states})")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("homeassistant_updates",
|
||||
sa.Column("reboot_after", sa.Boolean, nullable=False, server_default=sa.false()))
|
||||
op.add_column("homeassistant_updates", sa.Column("reboot_sent_at", sa.DateTime(timezone=True), nullable=True))
|
||||
_under_way("'backing_up', 'installing', 'rebooting'")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
_under_way("'backing_up', 'installing'")
|
||||
op.drop_column("homeassistant_updates", "reboot_sent_at")
|
||||
op.drop_column("homeassistant_updates", "reboot_after")
|
||||
@@ -63,12 +63,13 @@ class HomeAssistantUpdate(Base):
|
||||
moves through backing_up and installing to done or failed, and its row
|
||||
carries everything needed to carry on after Steward restarts. The *_before
|
||||
columns are Home Assistant's own readings at the start, so "a newer backup"
|
||||
is judged on its clock alone. One can be under way at a time.
|
||||
is judged on its clock alone. One can be under way at a time. `reboot_after`
|
||||
sends it through rebooting too, for an OS update the schedule installs.
|
||||
"""
|
||||
__tablename__ = "homeassistant_updates"
|
||||
__table_args__ = (
|
||||
Index("uq_homeassistant_update_under_way", text("(true)"), unique=True,
|
||||
postgresql_where=text("state IN ('backing_up', 'installing')")),
|
||||
postgresql_where=text("state IN ('backing_up', 'installing', 'rebooting')")),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||||
@@ -86,4 +87,6 @@ class HomeAssistantUpdate(Base):
|
||||
backup_event_before: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
backup_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
install_sent_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
reboot_after: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
reboot_sent_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
@@ -23,7 +23,7 @@ from steward.models.users import User, UserRole
|
||||
from . import client, schedule
|
||||
from .models import HomeAssistantConnection, HomeAssistantSchedule, HomeAssistantUpdate
|
||||
from .service import STALE_BACKUP_HOURS, backup_age_hours, backup_verdict, installable, parse_moment, problems_open
|
||||
from .updates import CannotStart, start_update
|
||||
from .updates import UNDER_WAY, CannotStart, start_update
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -72,7 +72,7 @@ async def _render_index(error: str | None = None, status: int = 200):
|
||||
installable=installable(connection),
|
||||
others=[u for u in updates if u["waiting"] and not u["kind"]],
|
||||
current=[u for u in updates if not u["waiting"] and u["kind"]],
|
||||
runs=runs, under_way=next((r for r in runs if r.state in ("backing_up", "installing")), None),
|
||||
runs=runs, under_way=next((r for r in runs if r.state in UNDER_WAY), None),
|
||||
backup=backup, backup_tone=tone, backup_says=backup_says,
|
||||
backup_at=parse_moment(backup.get("last_ok")), backup_next=parse_moment(backup.get("next")),
|
||||
backup_age=backup_age_hours(backup, now), stale_hours=STALE_BACKUP_HOURS,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# plugins/homeassistant/schedule.py
|
||||
"""Installing what's waiting at a weekly slot: one at a time through the same path
|
||||
as the page's Install, each after its own confirmed backup. Add-ons go first and
|
||||
the OS last, as it waits on a reboot. The first failure ends the slot, leaving the
|
||||
the OS last, as it reboots the machine. The first failure ends the slot, leaving the
|
||||
rest for someone to look at. Everything is on the rows, so a restart of Steward or
|
||||
of Home Assistant only delays the next step."""
|
||||
from __future__ import annotations
|
||||
@@ -76,7 +76,7 @@ async def step(app, transport=None) -> None:
|
||||
if update is None:
|
||||
return
|
||||
try:
|
||||
await start_update(app, update["entity_id"], ACTOR, transport)
|
||||
await start_update(app, update["entity_id"], ACTOR, transport, reboot_after=update["kind"] == "os")
|
||||
except CannotStart as exc:
|
||||
await _stop(app, schedule.id, f"{update['title']} wasn't started: {exc}. Nothing more was installed in this slot.")
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
<div id="ha-body"{% if under_way %} hx-get="/plugins/homeassistant/" hx-select="#ha-body" hx-trigger="every 15s" hx-swap="outerHTML"{% endif %}>
|
||||
<p class="page-verdict">
|
||||
{% if under_way %}{{ status_badge('pending', 'Updating') }}
|
||||
{{ under_way.title }}: {{ "taking a backup first" if under_way.state == 'backing_up' else "installing " ~ under_way.to_version }}. Home Assistant may stop answering for a while.
|
||||
{{ under_way.title }}: {{ {'backing_up': "taking a backup first", 'rebooting': "rebooting into " ~ under_way.to_version}.get(under_way.state, "installing " ~ under_way.to_version) }}. Home Assistant may stop answering for a while.
|
||||
{% elif reach == 'up' %}{{ status_badge('up', 'Answering') }}
|
||||
{% if installable %}{{ installable|length }} update{{ "s" if installable|length != 1 }} waiting.{% else %}Everything Steward can update is up to date.{% endif %}
|
||||
Read {{ connection.last_polled_at|ago }}.
|
||||
@@ -97,13 +97,13 @@
|
||||
<section class="card" aria-labelledby="ha-schedule">
|
||||
<h2 id="ha-schedule" class="section-title">Scheduled updates</h2>
|
||||
{% if plan and plan.enabled %}
|
||||
<p class="field-hint">{{ plan_when }}. Next: {{ plan_next|ago }}. Add-ons first, then Core, then the OS, each after a confirmed backup. A failure ends the run and the rest wait.</p>
|
||||
<p class="field-hint">{{ plan_when }}. Next: {{ plan_next|ago }}. Add-ons first, then Core, then the OS and its reboot, each after a confirmed backup. A failure ends the run and the rest wait.</p>
|
||||
{% else %}
|
||||
<p class="field-hint">Off. Updates are only installed when someone chooses Install.</p>
|
||||
{% endif %}
|
||||
{% if plan and plan.stopped %}<p class="tone-warn">The last run stopped early. {{ plan.stopped }}</p>{% endif %}
|
||||
{% if admin %}
|
||||
<p class="tone-warn">A Core update restarts Home Assistant with nobody there to watch, and an update can change how automations behave. An OS update takes effect only once you reboot the machine: Steward leaves that to you, and shows it under System until it's done.</p>
|
||||
<p class="tone-warn">A Core update restarts Home Assistant and an OS update reboots the machine, with nobody there to watch. An update can also change how automations behave.</p>
|
||||
<form method="post" action="/plugins/homeassistant/schedule" style="display:flex;flex-wrap:wrap;gap:0.75rem;align-items:flex-end;">{{ csrf_field() }}
|
||||
<label style="display:flex;gap:0.5rem;align-items:center;cursor:pointer;margin-bottom:0.4rem;">
|
||||
<input type="checkbox" name="enabled" value="on"{% if plan and plan.enabled %} checked{% endif %}> Install on a schedule
|
||||
@@ -137,7 +137,7 @@
|
||||
<td class="mono">{{ r.from_version or "—" }} → {{ r.to_version or "—" }}</td>
|
||||
<td>{% if r.state == 'done' %}{{ status_badge('up', 'Done') }}
|
||||
{% elif r.state == 'failed' %}{{ status_badge('down', 'Failed') }}<div class="cell-sub">{{ r.message }}</div>
|
||||
{% else %}{{ status_badge('pending', 'Taking a backup' if r.state == 'backing_up' else 'Installing') }}{% endif %}</td>
|
||||
{% else %}{{ status_badge('pending', {'backing_up': 'Taking a backup', 'rebooting': 'Rebooting'}.get(r.state, 'Installing')) }}{% endif %}</td>
|
||||
<td>{{ r.started_at|ago }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
|
||||
@@ -17,11 +17,13 @@ from .timeline import update_entry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
UNDER_WAY = ("backing_up", "installing")
|
||||
UNDER_WAY = ("backing_up", "installing", "rebooting")
|
||||
BACKUP_DEADLINE_S = 30 * 60
|
||||
REBOOT_DEADLINE_S = 20 * 60
|
||||
# An OS update downloads and writes a whole system image, which is slow on a Pi. Home
|
||||
# Assistant reports the new version as soon as that's written, before the reboot that
|
||||
# makes it run, and raises a repair asking for that reboot.
|
||||
# makes it run, and raises a repair asking for that reboot. The schedule's own OS
|
||||
# updates make that reboot; one started by a person leaves it to them.
|
||||
INSTALL_DEADLINE_S = {"addon": 20 * 60, "core": 30 * 60, "os": 45 * 60}
|
||||
|
||||
|
||||
@@ -42,10 +44,11 @@ async def under_way(session) -> HomeAssistantUpdate | None:
|
||||
HomeAssistantUpdate.state.in_(UNDER_WAY)))).scalars().first()
|
||||
|
||||
|
||||
async def start_update(app, entity_id: str, username: str, transport=None) -> HomeAssistantUpdate:
|
||||
async def start_update(app, entity_id: str, username: str, transport=None, *, reboot_after: bool = False) -> HomeAssistantUpdate:
|
||||
"""Start installing `entity_id`, an explicit action that may contact Home
|
||||
Assistant. Raises CannotStart, with nothing started, unless the update is
|
||||
still waiting and the signed-in user may install it."""
|
||||
still waiting and the signed-in user may install it. `reboot_after` reboots
|
||||
the machine once it's written, and it's done when the new version is running."""
|
||||
async with app.db_sessionmaker() as session:
|
||||
connection = (await session.execute(select(HomeAssistantConnection))).scalars().first()
|
||||
if connection is None:
|
||||
@@ -79,7 +82,7 @@ async def start_update(app, entity_id: str, username: str, transport=None) -> Ho
|
||||
entity_id=entity_id, title=known["title"], kind=known["kind"],
|
||||
from_version=attrs.get("installed_version"), to_version=attrs.get("latest_version"),
|
||||
started_by=username, backup_before=backup.get("state"),
|
||||
backup_event_before=(event or {}).get("state"))
|
||||
backup_event_before=(event or {}).get("state"), reboot_after=reboot_after)
|
||||
async with app.db_sessionmaker() as session:
|
||||
try:
|
||||
async with session.begin():
|
||||
@@ -117,7 +120,7 @@ async def _finish(app, run_id: str, state: str, message: str | None) -> None:
|
||||
logger.exception("Home Assistant update %s: its timeline entry could not be written", run_id)
|
||||
|
||||
|
||||
async def advance(app, transport=None) -> None:
|
||||
async def advance(app, transport=None, connect=None) -> None:
|
||||
"""Move the update under way one step on, if there is one."""
|
||||
async with app.db_sessionmaker() as session:
|
||||
run = await under_way(session)
|
||||
@@ -130,8 +133,10 @@ async def advance(app, transport=None) -> None:
|
||||
try:
|
||||
if run.state == "backing_up":
|
||||
await _advance_backup(app, run, connection, transport)
|
||||
else:
|
||||
elif run.state == "installing":
|
||||
await _advance_install(app, run, connection, transport)
|
||||
else:
|
||||
await _advance_reboot(app, run, connection, transport, connect)
|
||||
except client.Refused as exc:
|
||||
await _mark_refused(app, connection.id, str(exc))
|
||||
await _finish(app, run.id, "failed", str(exc))
|
||||
@@ -158,6 +163,10 @@ async def _past_deadline(app, run) -> bool:
|
||||
await _finish(app, run.id, "failed",
|
||||
f"Still not on {run.to_version} after {limit // 60} minutes. Look in Home Assistant.")
|
||||
return True
|
||||
if run.state == "rebooting" and _elapsed(run.reboot_sent_at or run.install_sent_at or run.started_at) > REBOOT_DEADLINE_S:
|
||||
await _finish(app, run.id, "failed", f"{run.to_version} was installed, but isn't running "
|
||||
f"{REBOOT_DEADLINE_S // 60} minutes after the reboot. Look in Home Assistant.")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@@ -207,6 +216,44 @@ async def _advance_install(app, run, connection, transport) -> None:
|
||||
installed = attrs.get("installed_version")
|
||||
if installed and (installed == run.to_version
|
||||
or (current.get("state") == "off" and installed != run.from_version)):
|
||||
if run.reboot_after:
|
||||
async with app.db_sessionmaker() as session:
|
||||
async with session.begin():
|
||||
row = await session.get(HomeAssistantUpdate, run.id)
|
||||
row.state = "rebooting"
|
||||
return
|
||||
await _finish(app, run.id, "done", None)
|
||||
return
|
||||
await _past_deadline(app, run)
|
||||
|
||||
|
||||
async def _mark_reboot_sent(app, run_id: str, at: datetime | None) -> None:
|
||||
async with app.db_sessionmaker() as session:
|
||||
async with session.begin():
|
||||
row = await session.get(HomeAssistantUpdate, run_id)
|
||||
row.reboot_sent_at = at
|
||||
|
||||
|
||||
async def _advance_reboot(app, run, connection, transport, connect) -> None:
|
||||
"""Reboot once, and wait for the new version to be what's running. The version
|
||||
is judged against the old one, so a machine that fell back to it fails at the
|
||||
deadline, as does one that never comes back."""
|
||||
token = await client.access_token(connection, transport)
|
||||
url, verify = connection.url, connection.verify_tls
|
||||
if run.reboot_sent_at is None:
|
||||
await _mark_reboot_sent(app, run.id, datetime.now(timezone.utc))
|
||||
try:
|
||||
await client.call_action(url, "hassio", "host_reboot", {}, token, verify, transport)
|
||||
except client.Unreachable:
|
||||
await _mark_reboot_sent(app, run.id, None)
|
||||
await _past_deadline(app, run)
|
||||
return
|
||||
except client.Rejected as exc:
|
||||
await _finish(app, run.id, "failed", f"{run.to_version} was installed, but Home Assistant "
|
||||
f"wouldn't reboot: {exc} It needs a reboot to take effect.")
|
||||
return
|
||||
found = await client.read_os(url, token, verify, connect)
|
||||
if found and found.get("version") not in (None, run.from_version) and not found.get("version_pending"):
|
||||
await _finish(app, run.id, "done", None)
|
||||
return
|
||||
await _past_deadline(app, run)
|
||||
|
||||
Reference in New Issue
Block a user