bvandeusenandClaude Opus 5.5
ecc59cc882
Home Assistant writes an OS update and leaves the reboot to someone, so a scheduled one never finished unattended. The schedule now asks for the reboot once, follows it through Home Assistant going away, and counts it done when the Supervisor reports a different version with nothing pending. A machine that doesn't come back or falls back to the old version fails after 20 minutes, which ends the slot and raises update_failed. Install on the page and through MCP still leave the reboot to the person. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
260 lines
13 KiB
Python
260 lines
13 KiB
Python
# plugins/homeassistant/updates.py
|
|
"""Installing one update: take a backup, confirm it landed, install, and follow
|
|
it to its end. Its row holds where it has got to, so Steward restarting, or Home
|
|
Assistant restarting as an update does, only delays the next step."""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.exc import IntegrityError
|
|
|
|
from . import client
|
|
from .models import HomeAssistantConnection, HomeAssistantUpdate
|
|
from .service import BACKUP_EVENT, LAST_BACKUP, is_newer, parse_moment
|
|
from .timeline import update_entry
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
UNDER_WAY = ("backing_up", "installing", "rebooting")
|
|
BACKUP_DEADLINE_S = 30 * 60
|
|
REBOOT_DEADLINE_S = 20 * 60
|
|
# An OS update downloads and writes a whole system image, which is slow on a Pi. Home
|
|
# Assistant reports the new version as soon as that's written, before the reboot that
|
|
# makes it run, and raises a repair asking for that reboot. The schedule's own OS
|
|
# updates make that reboot; one started by a person leaves it to them.
|
|
INSTALL_DEADLINE_S = {"addon": 20 * 60, "core": 30 * 60, "os": 45 * 60}
|
|
|
|
|
|
class CannotStart(Exception):
|
|
"""Why an update wasn't started, said to the person."""
|
|
|
|
|
|
async def _mark_refused(app, connection_id: str, said: str) -> None:
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
row = await session.get(HomeAssistantConnection, connection_id)
|
|
if row is not None:
|
|
row.refused, row.last_error = True, said
|
|
|
|
|
|
async def under_way(session) -> HomeAssistantUpdate | None:
|
|
return (await session.execute(select(HomeAssistantUpdate).where(
|
|
HomeAssistantUpdate.state.in_(UNDER_WAY)))).scalars().first()
|
|
|
|
|
|
async def start_update(app, entity_id: str, username: str, transport=None, *, reboot_after: bool = False) -> HomeAssistantUpdate:
|
|
"""Start installing `entity_id`, an explicit action that may contact Home
|
|
Assistant. Raises CannotStart, with nothing started, unless the update is
|
|
still waiting and the signed-in user may install it. `reboot_after` reboots
|
|
the machine once it's written, and it's done when the new version is running."""
|
|
async with app.db_sessionmaker() as session:
|
|
connection = (await session.execute(select(HomeAssistantConnection))).scalars().first()
|
|
if connection is None:
|
|
raise CannotStart("Home Assistant isn't connected")
|
|
if connection.refused:
|
|
raise CannotStart("Home Assistant refused Steward's sign-in")
|
|
if await under_way(session) is not None:
|
|
raise CannotStart("another update is under way")
|
|
if not connection.can_install:
|
|
raise CannotStart("the Home Assistant user who signed in isn't an administrator")
|
|
url, verify = connection.url, connection.verify_tls
|
|
known = next((u for u in connection.updates or [] if u["entity_id"] == entity_id), None)
|
|
if known is None or not known["kind"]:
|
|
raise CannotStart("that isn't a Core, OS or add-on update")
|
|
try:
|
|
token = await client.access_token(connection, transport)
|
|
current = await client.read_state(url, entity_id, token, verify, transport)
|
|
backup = await client.read_state(url, LAST_BACKUP, token, verify, transport)
|
|
event = await client.read_state(url, BACKUP_EVENT, token, verify, transport)
|
|
except client.Refused as exc:
|
|
await _mark_refused(app, connection.id, str(exc))
|
|
raise CannotStart(str(exc).rstrip(".")) from None
|
|
except client.Unreachable as exc:
|
|
raise CannotStart(str(exc).rstrip(".")) from None
|
|
if backup is None:
|
|
raise CannotStart("Home Assistant has no automatic backups to confirm one with")
|
|
if current is None or current.get("state") != "on":
|
|
raise CannotStart("that update is no longer waiting")
|
|
attrs = current.get("attributes") or {}
|
|
run = HomeAssistantUpdate(
|
|
entity_id=entity_id, title=known["title"], kind=known["kind"],
|
|
from_version=attrs.get("installed_version"), to_version=attrs.get("latest_version"),
|
|
started_by=username, backup_before=backup.get("state"),
|
|
backup_event_before=(event or {}).get("state"), reboot_after=reboot_after)
|
|
async with app.db_sessionmaker() as session:
|
|
try:
|
|
async with session.begin():
|
|
session.add(run)
|
|
await session.flush()
|
|
run_id = run.id
|
|
except IntegrityError:
|
|
raise CannotStart("another update is under way") from None
|
|
try:
|
|
await client.call_action(url, "backup", "create_automatic", {}, token, verify, transport)
|
|
except client.Refused as exc:
|
|
await _mark_refused(app, connection.id, str(exc))
|
|
await _finish(app, run_id, "failed", f"The backup couldn't be started: {exc} Nothing was installed.")
|
|
raise CannotStart(str(exc).rstrip(".")) from None
|
|
except (client.Unreachable, client.Rejected) as exc:
|
|
await _finish(app, run_id, "failed", f"The backup couldn't be started: {exc} Nothing was installed.")
|
|
raise CannotStart(str(exc).rstrip(".")) from None
|
|
async with app.db_sessionmaker() as session:
|
|
return await session.get(HomeAssistantUpdate, run_id)
|
|
|
|
|
|
async def _finish(app, run_id: str, state: str, message: str | None) -> None:
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
run = await session.get(HomeAssistantUpdate, run_id)
|
|
run.state, run.message, run.finished_at = state, message, datetime.now(timezone.utc)
|
|
# The page should show the new versions now, not at the next poll.
|
|
connection = (await session.execute(select(HomeAssistantConnection))).scalars().first()
|
|
if connection is not None:
|
|
connection.last_polled_at = None
|
|
try:
|
|
async with session.begin_nested():
|
|
await update_entry(session, run)
|
|
except Exception:
|
|
logger.exception("Home Assistant update %s: its timeline entry could not be written", run_id)
|
|
|
|
|
|
async def advance(app, transport=None, connect=None) -> None:
|
|
"""Move the update under way one step on, if there is one."""
|
|
async with app.db_sessionmaker() as session:
|
|
run = await under_way(session)
|
|
connection = (await session.execute(select(HomeAssistantConnection))).scalars().first()
|
|
if run is None:
|
|
return
|
|
if connection is None:
|
|
await _finish(app, run.id, "failed", "The Home Assistant connection was removed.")
|
|
return
|
|
try:
|
|
if run.state == "backing_up":
|
|
await _advance_backup(app, run, connection, transport)
|
|
elif run.state == "installing":
|
|
await _advance_install(app, run, connection, transport)
|
|
else:
|
|
await _advance_reboot(app, run, connection, transport, connect)
|
|
except client.Refused as exc:
|
|
await _mark_refused(app, connection.id, str(exc))
|
|
await _finish(app, run.id, "failed", str(exc))
|
|
except client.Rejected as exc:
|
|
await _finish(app, run.id, "failed", f"Home Assistant wouldn't install it: {exc}")
|
|
except client.Unreachable as exc:
|
|
# Expected while Home Assistant restarts: the next tick asks again.
|
|
logger.info("Home Assistant update %s: %s", run.id, exc)
|
|
await _past_deadline(app, run)
|
|
|
|
|
|
def _elapsed(since: datetime) -> float:
|
|
return (datetime.now(timezone.utc) - since).total_seconds()
|
|
|
|
|
|
async def _past_deadline(app, run) -> bool:
|
|
if run.state == "backing_up" and _elapsed(run.started_at) > BACKUP_DEADLINE_S:
|
|
await _finish(app, run.id, "failed",
|
|
f"No new backup within {BACKUP_DEADLINE_S // 60} minutes. Nothing was installed.")
|
|
return True
|
|
if run.state == "installing":
|
|
limit = INSTALL_DEADLINE_S.get(run.kind, INSTALL_DEADLINE_S["os"])
|
|
if _elapsed(run.install_sent_at or run.backup_at or run.started_at) > limit:
|
|
await _finish(app, run.id, "failed",
|
|
f"Still not on {run.to_version} after {limit // 60} minutes. Look in Home Assistant.")
|
|
return True
|
|
if run.state == "rebooting" and _elapsed(run.reboot_sent_at or run.install_sent_at or run.started_at) > REBOOT_DEADLINE_S:
|
|
await _finish(app, run.id, "failed", f"{run.to_version} was installed, but isn't running "
|
|
f"{REBOOT_DEADLINE_S // 60} minutes after the reboot. Look in Home Assistant.")
|
|
return True
|
|
return False
|
|
|
|
|
|
async def _advance_backup(app, run, connection, transport) -> None:
|
|
token = await client.access_token(connection, transport)
|
|
url, verify = connection.url, connection.verify_tls
|
|
backup = await client.read_state(url, LAST_BACKUP, token, verify, transport)
|
|
value = (backup or {}).get("state")
|
|
if is_newer(value, run.backup_before):
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
row = await session.get(HomeAssistantUpdate, run.id)
|
|
row.state, row.backup_at = "installing", parse_moment(value)
|
|
return
|
|
event = await client.read_state(url, BACKUP_EVENT, token, verify, transport)
|
|
if event and event.get("state") != run.backup_event_before \
|
|
and (event.get("attributes") or {}).get("event_type") == "failed":
|
|
reason = (event.get("attributes") or {}).get("failed_reason")
|
|
await _finish(app, run.id, "failed",
|
|
f"The backup failed{f' ({reason})' if reason else ''}. Nothing was installed.")
|
|
return
|
|
await _past_deadline(app, run)
|
|
|
|
|
|
async def _mark_sent(app, run_id: str, at: datetime | None) -> None:
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
row = await session.get(HomeAssistantUpdate, run_id)
|
|
row.install_sent_at = at
|
|
|
|
|
|
async def _advance_install(app, run, connection, transport) -> None:
|
|
token = await client.access_token(connection, transport)
|
|
url, verify = connection.url, connection.verify_tls
|
|
if run.install_sent_at is None:
|
|
# Marked first: Steward stopping here leaves an install that runs out its
|
|
# deadline, where asking twice would fail one that is going fine.
|
|
await _mark_sent(app, run.id, datetime.now(timezone.utc))
|
|
try:
|
|
await client.call_action(url, "update", "install", {"entity_id": run.entity_id}, token, verify, transport)
|
|
except client.Unreachable:
|
|
await _mark_sent(app, run.id, None)
|
|
raise
|
|
return
|
|
current = await client.read_state(url, run.entity_id, token, verify, transport)
|
|
attrs = (current or {}).get("attributes") or {}
|
|
installed = attrs.get("installed_version")
|
|
if installed and (installed == run.to_version
|
|
or (current.get("state") == "off" and installed != run.from_version)):
|
|
if run.reboot_after:
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
row = await session.get(HomeAssistantUpdate, run.id)
|
|
row.state = "rebooting"
|
|
return
|
|
await _finish(app, run.id, "done", None)
|
|
return
|
|
await _past_deadline(app, run)
|
|
|
|
|
|
async def _mark_reboot_sent(app, run_id: str, at: datetime | None) -> None:
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
row = await session.get(HomeAssistantUpdate, run_id)
|
|
row.reboot_sent_at = at
|
|
|
|
|
|
async def _advance_reboot(app, run, connection, transport, connect) -> None:
|
|
"""Reboot once, and wait for the new version to be what's running. The version
|
|
is judged against the old one, so a machine that fell back to it fails at the
|
|
deadline, as does one that never comes back."""
|
|
token = await client.access_token(connection, transport)
|
|
url, verify = connection.url, connection.verify_tls
|
|
if run.reboot_sent_at is None:
|
|
await _mark_reboot_sent(app, run.id, datetime.now(timezone.utc))
|
|
try:
|
|
await client.call_action(url, "hassio", "host_reboot", {}, token, verify, transport)
|
|
except client.Unreachable:
|
|
await _mark_reboot_sent(app, run.id, None)
|
|
await _past_deadline(app, run)
|
|
return
|
|
except client.Rejected as exc:
|
|
await _finish(app, run.id, "failed", f"{run.to_version} was installed, but Home Assistant "
|
|
f"wouldn't reboot: {exc} It needs a reboot to take effect.")
|
|
return
|
|
found = await client.read_os(url, token, verify, connect)
|
|
if found and found.get("version") not in (None, run.from_version) and not found.get("version_pending"):
|
|
await _finish(app, run.id, "done", None)
|
|
return
|
|
await _past_deadline(app, run)
|