bvandeusenandClaude Opus 5.5
f7a856cb90
The connection's job is keeping Home Assistant running. Each poll now reads open
repairs, Supervisor health, disk, recovery mode and add-ons that should be running
over its websocket, for the page, the overview and the problems_open, disk_free_pct
and update_failed metrics. That needs a websocket client, so websockets joins the
dependencies.
A weekly slot, set on the page or through MCP and off until then, installs add-ons,
then Core, then the OS, each after its own confirmed backup, ending at the first
failure. The slot arithmetic moves from the host agent's cadence to steward.core.weekly
so both share it.
The device scrape goes: the page's Metrics section and a target's token_from from
4c9ed02 are removed, and a migration deletes targets using it, as they'd otherwise ask
with no token. A refused token still stops a scrape.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
117 lines
5.4 KiB
Python
117 lines
5.4 KiB
Python
# plugins/homeassistant/scheduler.py
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.core.scheduler import ScheduledTask
|
|
|
|
from . import client, schedule
|
|
from .models import HomeAssistantConnection, HomeAssistantSchedule, HomeAssistantUpdate
|
|
from .service import RESOURCE, SOURCE, backup_age_hours, installable, read_health, read_snapshot
|
|
from .updates import advance
|
|
|
|
if TYPE_CHECKING:
|
|
from quart import Quart
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# An update under way is followed each tick, with one small read of its entity.
|
|
TICK_SECONDS = 15
|
|
# The whole entity list is a few hundred KB, and updates and backups change rarely.
|
|
POLL_SECONDS = 300
|
|
# A step past this is given up until the next tick, whatever the client's own
|
|
# timeouts did: a slow drip of bytes can keep each read inside its timeout for ever.
|
|
STEP_DEADLINE_SECONDS = 60
|
|
|
|
|
|
def make_tick_task(app: "Quart") -> ScheduledTask:
|
|
busy = asyncio.Lock()
|
|
|
|
async def tick() -> None:
|
|
if busy.locked():
|
|
return
|
|
async with busy:
|
|
for step in (poll, advance, schedule.step):
|
|
try:
|
|
await asyncio.wait_for(step(app), STEP_DEADLINE_SECONDS)
|
|
except TimeoutError:
|
|
logger.warning("Home Assistant %s gave up after %d s", step.__name__, STEP_DEADLINE_SECONDS)
|
|
|
|
return ScheduledTask(name="homeassistant_tick", coro_factory=tick,
|
|
interval_seconds=TICK_SECONDS, run_on_startup=True)
|
|
|
|
|
|
def is_due(connection, now: datetime) -> bool:
|
|
"""Whether to poll. Never after a refusal: another refused request counts
|
|
toward Home Assistant banning this address."""
|
|
if connection.refused:
|
|
return False
|
|
if connection.last_polled_at is None:
|
|
return True
|
|
return (now - connection.last_polled_at).total_seconds() >= POLL_SECONDS - TICK_SECONDS / 2
|
|
|
|
|
|
async def update_failed(session) -> bool:
|
|
"""Whether the newest update failed, or the schedule couldn't start one."""
|
|
stopped = (await session.execute(select(HomeAssistantSchedule.stopped))).scalars().first()
|
|
newest = (await session.execute(select(HomeAssistantUpdate.state).order_by(
|
|
HomeAssistantUpdate.started_at.desc()).limit(1))).scalars().first()
|
|
return bool(stopped) or newest == "failed"
|
|
|
|
|
|
async def poll(app: "Quart", transport=None, connect=None) -> None:
|
|
from steward.core.alerts import MetricBatch
|
|
|
|
now = datetime.now(timezone.utc)
|
|
async with app.db_sessionmaker() as session:
|
|
connection = (await session.execute(select(HomeAssistantConnection))).scalars().first()
|
|
if connection is None or not is_due(connection, now):
|
|
return
|
|
snapshot, health, error, refused = None, None, None, False
|
|
try:
|
|
token = await client.access_token(connection, transport)
|
|
snapshot = read_snapshot(await client.read_states(connection.url, token, connection.verify_tls, transport))
|
|
try:
|
|
health = read_health(await client.read_system(connection.url, token, connection.verify_tls, connect))
|
|
except client.Unreachable as exc:
|
|
# Updates and backups were read, so it's answering. The last report stands, and says it's old.
|
|
health = {**(connection.health or {}), "error": str(exc)}
|
|
except client.Refused as exc:
|
|
snapshot, error, refused = None, str(exc), True
|
|
except client.Unreachable as exc:
|
|
error = str(exc)
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
row = await session.get(HomeAssistantConnection, connection.id)
|
|
if row is None: # removed while it was being polled
|
|
return
|
|
batch = MetricBatch()
|
|
row.last_polled_at, row.last_error, row.refused = now, error, refused
|
|
if snapshot is None:
|
|
batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="is_up", value=0.0, recorded_at=now)
|
|
else:
|
|
row.last_ok_at = now
|
|
row.updates, row.backup, row.health = snapshot["updates"], snapshot["backup"], health
|
|
batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="is_up", value=1.0, recorded_at=now)
|
|
batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="updates_waiting",
|
|
value=len(installable(row)), recorded_at=now)
|
|
age = backup_age_hours(snapshot["backup"], now)
|
|
if age is not None:
|
|
batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="backup_age_hours",
|
|
value=age, recorded_at=now)
|
|
if not health.get("error"):
|
|
if health["repairs_read"]:
|
|
batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="problems_open",
|
|
value=len(health["problems"]), recorded_at=now)
|
|
if health["disk_free_pct"] is not None:
|
|
batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="disk_free_pct",
|
|
value=health["disk_free_pct"], recorded_at=now)
|
|
batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="update_failed",
|
|
value=1.0 if await update_failed(session) else 0.0, recorded_at=now)
|
|
await batch.record(session)
|