feat(hosts): the hottest sensor is named with its chip, and the host-reading cutoffs are read and set over MCP (#6806)

Agent 1.27.0 sends each temperature sensor's hwmon chip. Steward keeps the
chip per host and label beside the reading, so a sensor's metric key stays
its label, and the strip and the heat alert name it as "NVMe · Sensor 2"
rather than a bare "Sensor 2".

get_thresholds (viewer) and set_thresholds (admin, audited) read and change
the warn/crit cutoffs CPU, memory, disk, load and temperature are graded by,
as Settings → Thresholds does.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.forgejo/workflows/ci.yml

@@ -69,7 +69,12 @@ jobs:
           POSTGRES_USER: steward
           POSTGRES_PASSWORD: steward
           POSTGRES_DB: steward
+        # The data directory lives in RAM (#5746): postgres images declare
+        # VOLUME for it, so without a tmpfs each run left an anonymous volume
+        # that act_runner never removes, and 2,488 of them used up the runner
+        # host's inodes.
         options: >-
+          --tmpfs /var/lib/postgresql/data
           --health-cmd "pg_isready -U steward"
           --health-interval 10s
           --health-timeout 5s
@@ -168,6 +173,7 @@ jobs:
           IMAGE=git.fabledsword.com/bvandeusen/steward:${{ github.sha }}
           docker network create --internal "$SMOKE-net"
           docker run -d --name "$SMOKE-pg" --network "$SMOKE-net" \
+            --tmpfs /var/lib/postgresql/data \
             -e POSTGRES_USER=steward -e POSTGRES_PASSWORD=steward -e POSTGRES_DB=steward \
             postgres:16-alpine
           # Waits with deadlines (rule 156); pg_isready, not /dev/tcp, which is
@@ -206,6 +212,10 @@ jobs:
           # tini is PID 1, so zombies are reaped and SIGTERM reaches Steward.
           test "$(docker exec "$SMOKE-app" cat /proc/1/comm)" = "tini"
 
+          # The zone database is in the image, so a reader's zone resolves
+          # instead of quietly falling back to UTC (milestone 490).
+          docker exec "$SMOKE-app" python -c 'from zoneinfo import ZoneInfo; ZoneInfo("America/New_York")'
+
           # A fresh database lands on first-run setup: a real page, its assets
           # served by Steward itself, and no third-party host in it (rule 155).
           fetch / > /tmp/smoke-page.html
@@ -217,6 +227,49 @@ jobs:
             exit 1
           fi
 
+          # A backup made by the image restores into an empty database, and
+          # Steward starts on it (milestone 507). It carries settings (the
+          # marker setting) and leaves metrics and the audit log out.
+          sql() { docker exec "$SMOKE-pg" psql -U steward -v ON_ERROR_STOP=1 -tA "$@"; }
+          sql -d steward -c "INSERT INTO app_settings (key, value_json, updated_at) VALUES ('smoke.backup', '\"kept\"', now())"
+          sql -d steward -c "INSERT INTO plugin_metrics (id, source_module, resource_name, metric_name, value, recorded_at) VALUES ('smoke-metric', 'smoke', 'smoke', 'smoke', 1, now())"
+          sql -d steward -c "INSERT INTO audit_events (id, timestamp, username, action) VALUES ('smoke-audit', now(), 'smoke', 'smoke.backup')"
+          docker exec -u app "$SMOKE-app" steward backup --to /tmp/backups
+          # Checked against the database it was made from, it matches (#5823):
+          # a real pg_restore read, and what a running Steward writes by itself
+          # (last-seen times, backup history) left out of the comparison.
+          docker exec -u app "$SMOKE-app" sh -c 'steward check-backup /tmp/backups/steward-*.tar' | tee /tmp/smoke-check.txt
+          grep -q "Matches what is in place" /tmp/smoke-check.txt
+          # Put back in place over a changed setting (#5815): the setting comes
+          # back, and history the backup leaves out (the audit log) stays.
+          sql -d steward -c "UPDATE app_settings SET value_json = '\"changed\"' WHERE key = 'smoke.backup'"
+          docker exec -u app "$SMOKE-app" sh -c 'steward restore --in-place /tmp/backups/steward-*.tar'
+          test "$(sql -d steward -c "SELECT value_json FROM app_settings WHERE key = 'smoke.backup'")" = '"kept"'
+          test "$(sql -d steward -c "SELECT count(*) FROM audit_events WHERE id = 'smoke-audit'")" = "1"
+          sql -d steward -c "CREATE DATABASE restored"
+          docker exec -u app \
+            -e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/restored" \
+            "$SMOKE-app" sh -c 'steward restore /tmp/backups/steward-*.tar'
+          test "$(sql -d restored -c "SELECT value_json FROM app_settings WHERE key = 'smoke.backup'")" = '"kept"'
+          test "$(sql -d restored -c "SELECT count(*) FROM plugin_metrics")" = "0"
+          test "$(sql -d restored -c "SELECT count(*) FROM audit_events")" = "0"
+          test "$(sql -d restored -c "SELECT count(*) FROM alembic_version")" -ge 1
+          docker run -d --name "$SMOKE-restored" --network "$SMOKE-net" \
+            -e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/restored" \
+            -e STEWARD_SECRET_KEY=smoke-test-placeholder \
+            "$IMAGE"
+          up=""
+          for _ in $(seq 1 60); do
+            if docker exec "$SMOKE-restored" python -c 'import urllib.request
+          urllib.request.urlopen("http://127.0.0.1:5000/health", timeout=5)' >/dev/null 2>&1; then up=1; break; fi
+            sleep 2
+          done
+          if [ -z "$up" ]; then
+            echo "FATAL: Steward never started on the restored database"
+            docker logs "$SMOKE-restored" 2>&1 | tail -80
+            exit 1
+          fi
+
           # A stop is a clean shutdown: SIGTERM through tini to Hypercorn, which
           # finishes within its graceful timeout and exits 0.
           docker stop -t 30 "$SMOKE-app"
@@ -242,7 +295,8 @@ jobs:
       - name: Smoke clean-up
         if: always()
         run: |
-          docker rm -f "steward-smoke-${{ github.run_id }}-app" "steward-smoke-${{ github.run_id }}-pg" >/dev/null 2>&1 || true
+          docker rm -fv "steward-smoke-${{ github.run_id }}-app" "steward-smoke-${{ github.run_id }}-restored" \
+            "steward-smoke-${{ github.run_id }}-pg" >/dev/null 2>&1 || true
           docker network rm "steward-smoke-${{ github.run_id }}-net" >/dev/null 2>&1 || true
       - name: Prune dangling layers
         if: always()

Dockerfile

@@ -16,6 +16,12 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \
         # tini — PID 1: reaps the zombies ssh and ansible leave behind and
         # forwards SIGTERM to Steward so a stop is a graceful shutdown
         tini \
+        # tzdata — the zone database, so times show in each reader's own zone
+        # (milestone 490); Debian's slim base no longer carries it
+        tzdata \
+        # postgresql-client-17 — pg_dump and pg_restore for Steward's own
+        # backups (milestone 507); 17 reads and writes a Postgres 16 server
+        postgresql-client-17 \
     && rm -rf /var/lib/apt/lists/*
 
 # Upgrade pip to suppress the version notice
@@ -46,7 +52,7 @@ RUN useradd -m -u 1000 app \
     && mkdir -p /data/ansible/sources /data/ansible/runs /data/ansible/playbooks /data/plugins \
     && chown -R app:app /data /app
 
-EXPOSE 5000
+EXPOSE 5000 5514/udp 5514/tcp
 
 ENTRYPOINT ["/usr/bin/tini", "--", "/entrypoint.sh"]
 CMD ["steward", "--host", "0.0.0.0", "--port", "5000"]

README.md

@@ -53,7 +53,7 @@ cp .env.example .env
 docker compose -f compose.deploy.yml up -d
 ```
 
-Open `http://<server>:5000` and create the admin account on first run.
+Open `http://<server>:5000` and create the admin account on first run. Devices send syslog to `<server>:5514` (UDP or TCP); see `docs/plugins/syslog.md`.
 
 **Update to the latest dev build:**
 
@@ -119,7 +119,14 @@ See `docs/plugins/` for a full plugin development guide.
 | `docs/core/monitors.md` | Ping and DNS monitors: probe logic, metrics emitted, data models |
 | `docs/core/alerting.md` | Alert rules, state machine, notification channels, template variables |
 | `docs/core/ansible.md` | Playbook sources, run lifecycle, SSE streaming |
+| `docs/core/backups.md` | Steward's daily backup to an S3 bucket (Fabled Archives), what it holds, and `steward restore` |
+| `docs/core/timeline.md` | The timeline: what it records from each source, areas and kinds, Settings → Timeline |
+| `docs/core/tickets.md` | Tickets: known problems, linked to alerts and anomalies, shown as known issues, resolved onto the timeline |
 | `docs/plugins/overview.md` | Plugin system: how loading works, plugin.yaml schema, required exports |
 | `docs/plugins/writing-a-plugin.md` | Step-by-step plugin development guide |
 | `docs/plugins/traefik.md` | Traefik plugin: config, metrics, alert rule examples |
+| `docs/plugins/prometheus.md` | Prometheus plugin: scraping any /metrics endpoint, how series are stored, the Fabled Archives recipe |
+| `docs/plugins/homeassistant.md` | Home Assistant plugin: waiting updates, backup age, and installing updates after a confirmed backup |
+| `docs/plugins/syslog.md` | Syslog plugin: sources, patterns and alerts, publishing port 5514 (ingress or host mode under Swarm) |
+| `docs/plugins/prometheus-widgets.md` | Prometheus widget templates: writing, sharing and copying one in, every block, the format reference |
 | `docs/reference/code-map.md` | Where every key function, model, and route lives in the codebase |

compose.deploy.yml

@@ -16,6 +16,14 @@ services:
     pull_policy: always
     ports:
       - "5000:5000"
+      # Syslog (plugins/syslog): devices send here. Steward runs unprivileged,
+      # so it listens above 1024; publish 514 to it instead ("514:5514/udp")
+      # if a device can't choose its port. Under Docker Swarm's ingress every
+      # device arrives from one internal address and is told apart by the name
+      # its lines carry; the long form with `mode: host` keeps each device's
+      # own address but ties Steward to that node.
+      - "5514:5514/udp"
+      - "5514:5514/tcp"
     networks:
       - backend
     volumes:
@@ -39,8 +47,7 @@ services:
       # mount, pin STEWARD_SECRET_KEY (env or a docker secret) instead, so the key
       # never depends on filesystem ownership being correct on every node.
     depends_on:
-      db:
+      - db
-        condition: service_healthy
     restart: unless-stopped
 
   db:
@@ -54,11 +61,9 @@ services:
     volumes:
       - pgdata:/var/lib/postgresql/data
     # No host port published — db is reachable only on the backend network.
-    healthcheck:
+    # No healthcheck: a slow disk (NFS) failed pg_isready and the orchestrator
-      test: ["CMD-SHELL", "pg_isready -U steward"]
+    # killed a healthy database mid-write (#5275). Steward waits for the
-      interval: 5s
+    # database itself, at startup and per request (steward/database.py).
-      timeout: 5s
-      retries: 5
     restart: unless-stopped
 
 networks:

docker-compose.yml

@@ -8,6 +8,8 @@ services:
     command: ["steward", "--host", "0.0.0.0", "--port", "5000", "--debug"]
     ports:
       - "5000:5000"
+      - "5514:5514/udp"     # syslog (plugins/syslog)
+      - "5514:5514/tcp"
     volumes:
       - app_data:/data
       # Live-edit core + first-party plugins without rebuilding. PYTHONPATH=/app
@@ -27,8 +29,7 @@ services:
       - STEWARD_SECRET_KEY=dev-secret-key-change-me
       - PYTHONPATH=/app
     depends_on:
-      db:
+      - db
-        condition: service_healthy
     restart: unless-stopped
 
   db:
@@ -42,11 +43,9 @@ services:
       - "5432:5432"
     volumes:
       - pgdata:/var/lib/postgresql/data
-    healthcheck:
+    # No healthcheck: a slow disk (NFS) failed pg_isready and the orchestrator
-      test: ["CMD-SHELL", "pg_isready -U steward"]
+    # killed a healthy database mid-write (#5275). Steward waits for the
-      interval: 5s
+    # database itself, at startup and per request (steward/database.py).
-      timeout: 5s
-      retries: 5
     restart: unless-stopped
 
 volumes:

docs/architecture.md

@@ -87,6 +87,48 @@ Core tasks registered in `app._task_registry`:
 
 ---
 
+## Metric Store
+
+Every plugin and monitor records into one store keyed by (source, resource, metric):
+- `plugin_metrics` holds raw samples for `metrics.retention.raw_days` (7 by default).
+- `plugin_metric_latest` holds each key's newest value.
+- `data_cleanup` rolls raw samples older than the raw window into `plugin_metrics_hourly`, kept for `metrics.retention.rollup_days` (90 by default).
+
+Each rolled-up hour keeps three figures:
+- `value_avg`: the hour's average.
+- `value_max`: its peak.
+- `value_p95`: the level it held for all but its briefest 5%, about 3 minutes.
+
+A two-minute spike raises the peak and leaves the p95 low; an hour spent at that level raises both. Learned normals use this to tell a recurring spike from a held level. Hours rolled up before the p95 was kept have it NULL; read those as their average.
+
+### Normals and anomalies
+
+A metric a plugin registers with `learn_normal()` gets a normal per hour of the day, then per hour of the week once three weeks of history exist (`metric_norms`, relearned hourly by `metric_norms`; `steward/core/norms.py`).
+
+Every minute, `metric_anomalies` averages each learned key's last 15 minutes and compares it with the normal for that hour. A level above the usual hold ×1.25, and above the metric's floor, has departed. It opens an anomaly once it is also higher than the metric usually goes, or held: the median of the last hour is over the same line. How high it usually goes is the greater of the usual peak for that hour and how high it has gone in most hours of the day, so bursts that land at any hour are told from a job with a time of its own. A burst of a few minutes within that lifts the 15-minute average and opens nothing. Height is judged with the window's highest reading left out, so one slow request can't carry a window past it, and a held level needs at least three readings behind its median, so a lone reading never opens an anomaly (`steward/core/anomalies.py`):
+- `metric_anomalies` holds one row per item to review. A repeat while the row still waits joins it as another occurrence (`metric_anomaly_spans`).
+- A row is `to_review` until someone marks it `normal` or `real` (Alerts → Anomalies, or MCP `review_anomaly`). An ended row nobody reviews within `anomalies.review_days` (14) becomes `lapsed`.
+- Learning skips every hour an occurrence covers unless its anomaly was marked `normal`, so a new level is never learned without review.
+- `data_cleanup` prunes reviewed and lapsed rows at `data.retention_days`; a row still to review is never pruned.
+
+What learns a normal, and the floor each departure must clear (Settings → Thresholds → Normals, `norms.floor.*`; a floor may be a function, `floor_setting()`, read at each pass):
+
+| Kind | Metric | Floor (default) |
+|---|---|---|
+| Host CPU | `host_agent` / `cpu_pct` | 10 points |
+| Host memory | `host_agent` / `mem_used_pct` | 5 points |
+| Host load per core | `host_agent` / `load_pct` (load_1m ÷ cores × 100, recorded at ingest) | 10 points |
+| Container CPU | `docker` / `cpu_pct` (100% is one core; a swarm service's under `swarm/<service>`, shared by its tasks) | 10 points of one core |
+| Container memory | `docker` / `mem_used_pct` (its processes' own memory, file cache left out, of its limit or of host RAM when it has none) | 5 points |
+| Monitor response time | `<type>` / `response_ms` | 100 ms |
+| Router 4xx share, p95 | `traefik` / `error_rate_4xx_pct`, `latency_p95_ms` | 2 points, 50 ms |
+
+Host metrics learn with `parts=False`: the host learns, its cores (`<host>:core<N>`) don't. Disk space, a monitor that is down, an agent gone quiet and a router's 5xx share are failures and keep their fixed cutoffs (a router's shares are judged only over enough requests).
+
+Pages grade by the normal through `anomalies.graded()` (hosts' `host_grades` and `worst_reading(normals=)`, `docker.service.usage(thresholds=, norms=)`, `monitors.service.latency_grade`): a reading past its cutoff but within its normal reads healthy and says what is usual at this hour. The host, monitor, container and Traefik pages each carry a one-line `_anomaly_card.html`. It says how many anomalies wait for review and names any that are out of their normal now. Its Review button opens Alerts → Anomalies narrowed to that resource (`?source=&resource=`), where they are reviewed.
+
+---
+
 ## Database Session Pattern
 
 `app.db_sessionmaker` is an SQLAlchemy `async_sessionmaker`. All DB access follows this pattern:

docs/core/alerting.md

@@ -32,18 +32,25 @@ Notifications are sent via `asyncio.create_task()` after the function returns, s
 Each `AlertRule` has one associated `AlertState` row. The state transitions are:
 
 ```
-inactive ──(breached)──► pending ──(consecutive count met)──► FIRING
+inactive ──(breached)──► pending ──(N breaching checks in a row)──► FIRING
-                │                                               │
+    ▲                      │                                          │
-                └──(recovered)──► inactive (no notification)   │
+    └──(an OK check)───────┘                                          │
-                                                               │
-                                          FIRING ──(recovered)──► RESOLVED ──► inactive
                                  FIRING ──(acknowledged)──► ACKNOWLEDGED
-                                          ACKNOWLEDGED ──(recovered)──► RESOLVED ──► inactive
+                                 FIRING / ACKNOWLEDGED ──(M OK checks in a row)──► RESOLVED ──► inactive
-                                          ACKNOWLEDGED ──(re-breached)──► FIRING
 ```
 
+N is the rule's consecutive checks, M its checks to clear. An acknowledgement
+holds for as long as the breach lasts: only a resolve clears it, and a breaching
+check never sends an acknowledged alert back to firing.
+
 `RESOLVED` is transient — the evaluator writes the event and immediately sets state back to `inactive` within the same transaction. `RESOLVED` never persists as a final state in the DB.
 
+Every transition is an `AlertEvent`, the history the Alerts → History page and
+MCP `get_alert_history` read. A transition a person made carries them in
+`actor_id`: an acknowledgement (firing → acknowledged), or a rule edit, disable
+or enable that reset the alert (→ resolved, with a note such as "rule
+disabled"). Transitions caused by readings, or by Steward itself, have none.
+
 ### State Definitions
 
 | State | Meaning |
@@ -51,7 +58,7 @@ inactive ──(breached)──► pending ──(consecutive count met)──
 | `inactive` | Threshold not breached |
 | `pending` | Threshold breached but consecutive count not yet met; no notification sent |
 | `firing` | Consecutive count met; FIRING notification sent |
-| `acknowledged` | Operator acknowledged; suppresses repeat notifications; auto-clears on recovery |
+| `acknowledged` | Someone has seen it; no further notifications; clears when the readings recover |
 | `resolved` | Transient — notification sent, immediately transitions to `inactive` |
 
 ---
@@ -66,8 +73,42 @@ Alert rules are created in the UI at `/alerts/`. Each rule requires:
 - **Metric name** — the metric key (e.g. `response_time_ms`, `up`, `error_rate_5xx_pct`)
 - **Operator** — `>`, `<`, `>=`, `<=`, `==`, `!=`
 - **Threshold** — numeric value
+- **rose by** — instead of a line to cross, fires when a reading is at least the threshold above the one before it, for a count that only grows (a disk's bad sectors). A fall doesn't fire. A reading that hasn't risen counts as OK, so the checks to clear set how long the alert stays up, and rises inside that are one alert
 - **Consecutive failures required** — how many consecutive breaches before FIRING (default 1)
 
+The form offers only what has reported, read from `plugin_metric_latest`.
+For the host agent, a **Host** is picked first, then a **Part** of it: "The
+whole host", then its mounts, CPU cores, disks and so on, under a heading for
+each kind. A part that reports nothing a rule can watch, such as an interface's
+traffic, isn't listed. The **Metric** list holds only what the picked resource
+reports. Under it, a preview shows the metric's current reading and its last
+24 hours (lowest to highest, with an hourly sparkline; it loads just after the
+page, since a day of samples can take a while to read), so the threshold can be
+set against real numbers.
+
+### Rules that cover hosts
+
+A rule on `host_agent` can cover **every host**, or **every host in an
+inventory group**, instead of naming one resource (#5464). Inventory groups
+are Steward's grouping system: a host is in a group through its linked
+Ansible target.
+
+A covering rule has no state of its own. Steward keeps one ordinary rule per
+covered host under it, named `<rule>: <host>`, so each host's alert keeps its
+own state, history and acknowledgement. Each minute (the `alert_reconcile`
+task), and whenever the rule is saved:
+
+- a host that has reported in the last 7 days, and is covered, gets a rule;
+- a host that is no longer covered loses its rule, and that rule's history;
+- the hosts' rules take the covering rule's condition, checks, notification
+  setting, action and on/off.
+
+A host's rule is changed through its covering rule; editing, disabling or
+deleting it alone is refused. What a rule covers is fixed once it exists, but
+a group rule can move to another group. Over MCP, `create_alert_rule` takes
+`covers` ("all_hosts" or "group") and `group`, and `list_alert_rules` shows
+each covering rule's `hosts` and each host's rule's `part_of`.
+
 ### Available Metrics by Module
 
 | `source_module` | `metric_name` | Description |
@@ -83,6 +124,25 @@ Alert rules are created in the UI at `/alerts/`. Each rule requires:
 | `traefik` | `latency_p95_ms` | Approximate p95 latency (ms) |
 | `traefik` | `latency_p99_ms` | Approximate p99 latency (ms) |
 
+#### Steward itself
+
+Source `steward`, resource `process`, recorded once a minute by the
+self-monitor tick (`steward/core/self_monitor.py`; the ingest tally is in
+`steward/core/health.py`). Settings → System (`/settings/system/`, admin)
+shows each one now and over the last day, with a verdict above that names
+failed agent posts, a gap in watching, or open files near the limit.
+
+| `metric_name` | Description |
+|---|---|
+| `ingest_posts` | Host-agent posts Steward took in the last minute |
+| `ingest_failed` | Posts in the last minute that didn't land (handler raised, or the agent gave up) |
+| `ingest_slowest_ms` | The slowest post in the last minute; agents give up at 10 s |
+| `db_roundtrip_ms` | A `SELECT 1`, connection checkout included |
+| `unwatched_secs` | The gap before this tick when it exceeds `watch.GAP_S` (a deploy or an outage); 0 when unbroken |
+| `restart_gap_secs` | Recorded once, on a new process's first tick: the time since the last process's last sample, however short |
+| `cleanup_secs` | How long the last hourly retention and rollup run took |
+| `open_fds` / `open_fds_pct` | Steward's open files, and as a share of its limit (absent without `/proc`) |
+
 ---
 
 ## Notifications
@@ -104,6 +164,43 @@ Configured at `/settings/` under SMTP. Settings:
 
 Email is skipped if `smtp.host` is empty.
 
+Alerts that arrive together go out as one email. The first alert opens a
+2-minute window. When it closes, each recipient gets one message listing every
+alert in the window, oldest first, in their own zone, with a subject like
+"[Steward] 3 alerts: 2 firing, 1 resolved". One alert on its own keeps its
+usual email.
+
+Every email Steward sends (alerts, the weekly report, invites, password
+resets and the test buttons) waits in the database (`email_outbox`,
+`core/outbox.py`) until the mail server accepts it, so nothing waiting is lost
+when Steward restarts: the sender picks it up at boot. The sender runs every 15
+seconds and settles each email on the server's answer:
+
+- **Accepted:** sent.
+- **Refused** (an SMTP 5xx, such as "no such mailbox"): failed at once, with
+  the server's words. Trying again would only repeat the refusal.
+- **Temporary** (no connection, a timeout, a 4xx): tried again after 30 s,
+  1, 2, 5, 10 and then every 15 minutes. An email the server still hasn't
+  taken an hour after its first try is failed with the last error.
+
+Once every email carrying an alert has settled, that alert's event records
+whether the email went, and to whom it didn't.
+
+Where to see it: each alert's row in **Alerts → History** says what its email
+is doing (gathering, with when it sends; retrying, with the attempt, the next
+try and the server's last answer; sent; failed, with who wasn't reached and
+why), and `get_alert_history` over MCP reports the same. **Settings → System →
+Outgoing email** lists every email of the last 7 days not yet sent, failures
+first, and the page's verdict names failures and any email still retrying
+after 5 minutes.
+
+An email a person is waiting on is tried at once instead of on the next tick:
+the weekly report's **Send now**, an invite, and the test buttons, so the page
+can say what the server answered. A report or invite the server couldn't take
+yet stays queued and is retried as above. A test (the Settings → Notifications
+test email, a rule's **Send test**) is recorded but never retried: its one
+answer is the result.
+
 ### Webhook
 
 Configured at `/settings/` under Webhook. The body is a Jinja2 template rendered to JSON. Content-Type is always `application/json`. If the rendered template is not valid JSON, the delivery is logged as failed and no request is sent.
@@ -117,6 +214,7 @@ Configured at `/settings/` under Webhook. The body is a Jinja2 template rendered
 | `alert.threshold` | float | Configured threshold |
 | `alert.resource` | str | Resource name |
 | `alert.source_module` | str | `ping`, `dns`, `traefik`, etc. |
+| `alert.which` | str | For a figure that counts things, what it counts when it fires, joined by ", " (`web_app (0 of 2 running), …`); `raw.which` is the list. Empty for other figures. |
 | `alert.timestamp` | str | ISO 8601 UTC |
 
 Default template (Discord-compatible):
@@ -124,7 +222,29 @@ Default template (Discord-compatible):
 {"content": "**{{ alert.state }}** — {{ alert.resource }} — {{ alert.rule_name }} ({{ alert.metric }} = {{ alert.value }})"}
 ```
 
-Webhook is skipped if `webhook.url` is empty.
+Webhook is skipped if `webhook.url` is empty. Webhooks are not gathered into
+bursts: each alert posts on its own, at once.
+
+### Send test
+
+**Send test** at the top of a rule's page (operator role), or `send_test_alert` over MCP,
+sends a `TEST` alert for that rule now, by email and webhook, the same way a
+real alert would go. It uses the rule's name and condition and the latest
+reading of what the rule watches. A rule that covers hosts uses its first
+host. Each channel's result shows under the button. Nothing fires, the rule's
+history doesn't change, and the test is recorded in the audit log as
+`alert_rule.tested`.
+
+### On the dashboard
+
+The **Alerts** widget (`/alerts/widget`) sits beside the Overview in the
+recommended layout. It opens with a verdict ("2 firing, 1 silent." or
+"Nothing firing: all 12 rules are quiet."). Below that it lists what is
+firing or silent: each rule, what it watches, its reading now against what
+it fires on, and since when. Acknowledged alerts follow, with who
+acknowledged them and when. A line appears when alert email has failed in
+the last 7 days or is still retrying, linking admins to Settings → System.
+The quiet rules are counted at the end.
 
 ---
 
@@ -135,5 +255,5 @@ Defined in `steward/models/alerts.py`:
 - **`alert_rules`** — one row per configured rule
 - **`alert_states`** — one row per rule, tracks current state and consecutive failure count
 - **`alert_events`** — append-only log of all state transitions and notification outcomes
-- **`plugin_metrics`** — all metric values written by any monitor or plugin
+- **`plugin_metrics`** — all metric values written by any monitor or plugin. Its series index carries the value (`ix_plugin_metrics_series_value`), so a day of one metric is read from the index alone even when nothing is cached; Steward builds it in the background after startup (`core/metric_index.py`), never during a migration
 - **`plugin_metric_latest`** — the newest sample of each (source, resource, metric). Pages that show "what it is now" read this instead of searching history; retention drops a row once its sample leaves the raw window

docs/core/backups.md

@@ -0,0 +1,220 @@
+# Backups: Steward backing itself up, and coming back
+
+Steward sends a backup of itself to an S3 bucket once a day. Fabled Archives
+is the intended target. A backup is one file. Settings → Backups restores it
+on a running Steward, first-run setup restores it on a new one, and
+`steward restore` brings Steward back from it when Steward can't start.
+
+## What a backup holds
+
+`steward-<UTC time>.tar`, containing:
+
+| Member | What it is |
+|---|---|
+| `steward.dump` | `pg_dump --format=custom` of Steward's database |
+| `secret.key` | The key Steward's stored passwords and tokens are sealed with |
+| `manifest.json` | Format, Steward version, migration heads, time, and the tables whose rows were left out |
+
+A backup is for **system recovery**: it holds what you would otherwise have
+to set up again, and no logs or history.
+
+**Kept:**
+- settings, with the passwords and keys they hold;
+- users and API tokens;
+- hosts and their agent enrolments, so agents don't need enrolling again;
+- monitors, alert rules, maintenance windows and maintenance cadences;
+- Ansible inventory and schedules;
+- dashboards and share links;
+- SNMP devices, Prometheus targets and widget templates, UniFi client names.
+
+**Left out** (the tables come back empty and fill again as Steward runs):
+- the audit log;
+- alert history and alert state. Each rule starts inactive and is watched
+  again within a minute;
+- metric anomalies and their reviews;
+- metrics, their rollups and learned normals;
+- monitor check results;
+- Docker, Traefik and UniFi history;
+- Ansible run records and maintenance job history;
+- the email queue, pending invites and password-reset links.
+
+The list is `steward/backup/contents.py`. A test fails when a new table
+appears in neither list, so whoever adds a table decides.
+
+**The secret key is in the file.** That makes a restore complete on its own.
+It also means anyone holding a copy can read every password and key Steward
+keeps. Keep the bucket private.
+
+## Setting it up with Fabled Archives
+
+1. In Archives, make a backup target for Steward with an S3 key. The default
+   append-only key is enough: Steward only uploads and never deletes.
+   How long backups are kept is set on the Archives side.
+2. In Steward, open **Settings → Backups** and fill in:
+   - **Endpoint**: the S3 address Archives shows for the target, without the
+     bucket (for example `http://archives:8081`). Requests are path-style:
+     `endpoint/bucket/file`.
+   - **Bucket**: the target's bucket name.
+   - **Access key** and **Secret key**: from Archives. The secret is sealed at
+     rest; a blank box on save keeps the stored one.
+3. Press **Test the bucket**. A key that may not list ("AccessDenied") still
+   counts as reached: an append-only key can't list, and uploads work anyway.
+4. Tick **Back up every day**, pick the hour (on your clock), and save.
+   **Back up now** makes one straight away.
+
+A day missed while Steward was down is made up when it comes back. A failed
+backup is tried again an hour later, never sooner. The last 30 runs are
+listed on the page, and each one is in the audit log (`backup.made` /
+`backup.failed`).
+
+### Alert when backups stop
+
+While backups are on, Steward records two metrics under the source
+**Steward itself**, resource `backup`:
+
+| Metric | Meaning |
+|---|---|
+| `backup_age_secs` ("Newest backup's age") | Seconds since the last backup was sent, updated every 15 minutes |
+| `backup_bytes` ("Backup size") | The size of the last backup sent |
+
+A rule such as `backup_age_secs > 93600` (26 hours) fires when a daily backup
+has been missed.
+
+An agent on the MCP sees the same without a rule. `get_overview` names a
+scheduled backup that failed, is two hours past its slot with none made, or has
+no bucket set up. `list_backups`, `check_backup` and `make_backup` are this
+page's admin tools (see [MCP](mcp.md)); restoring isn't among them.
+
+## Checking a backup
+
+**Settings → Backups → Check** compares a backup with what is in place, without
+changing anything. It shows what a restore would bring back, change or remove.
+Each row under Recent backups has its own Check button, and **Check a file**
+takes a file downloaded from the bucket. From a shell:
+
+```sh
+docker exec -u app <steward container> steward check-backup /tmp/steward-20261015T030000Z.tar
+```
+
+The check reports:
+- for each area (Settings, Users and API tokens, Hosts and agents, Monitors,
+  Alert rules, Ansible, Dashboards, Plugins), how many entries are the same,
+  differ, are only in the backup or are only here;
+- each setting that differs, with both values. A password or key shows only
+  "differs": sealed values are opened with their own key and compared, never
+  shown;
+- whether the backup's secret key is the one in use, and whether it opens the
+  backup's own sealed values. A backup whose key can't open them fails the
+  check, because its secrets would not come back;
+- whether the backup was made by this Steward's schema, an older one or a
+  newer one.
+
+What Steward records as it runs is left out of the comparison: when an agent
+or monitor was last seen, a schedule's last run, the backup history. These
+change by the minute. The list is `ACTIVITY` and `STATE_SETTINGS` in
+`steward/backup/contents.py`.
+
+Each backup Steward sends is checked once it is written, and its row under
+Recent backups says so. A backup is sent even when its check can't run.
+
+Checking a backup on the bucket reads it back from there. An append-only key
+may not be allowed to; the page then says AccessDenied, and the file can be
+downloaded and checked with **Check a file** instead.
+
+## Making a backup by hand
+
+```sh
+docker exec -u app <steward container> steward backup --to /tmp/backups
+```
+
+This writes the file and sends it nowhere. It needs the same database access
+as Steward itself.
+
+## Restoring
+
+### From the web UI
+
+**On a running Steward: Settings → Backups.** Press **Restore** on one of
+the recent backups, or choose a file on the "Check or restore a file" card.
+Steward first shows what the backup would change: the same comparison as
+Check. Then it says what a restore does and asks you to type `restore`.
+
+- **A backup of this Steward's schema goes back in place.** Settings,
+  users, hosts, monitors, alert rules and dashboards become the backup's.
+  The audit log and the history of everything still here stay. History of
+  anything the backup doesn't have goes with it. It is one transaction: if
+  anything fails part-way, nothing changes, and Backups says why.
+- **A backup from an older Steward replaces everything**, history included,
+  and you tick "Replace everything" as well. Steward brings it up to date
+  as it starts.
+- **A backup from a newer Steward isn't offered.** Update Steward first.
+
+Before anything changes, Steward makes a **safety backup** of itself as it
+stands. It goes to `/data/backups` (the newest five are kept) and to the
+bucket, and it is listed under Recent backups as "before restore". A
+restore can be undone by restoring that.
+
+While the restore runs:
+- every page says Steward is restoring;
+- agents get a 503 and keep their readings until Steward is back;
+- scheduled work holds.
+
+Then Steward restarts (it exits with code 3 so Swarm, compose or systemd
+starts it again), and you sign in as a user from the backup. If an older
+backup's full replace fails after the database was emptied, Steward starts
+at first-run setup, where the safety backup can be uploaded.
+
+**On a new Steward: first-run setup.** Next to "Create the initial admin
+account" is **Or restore from a backup**: upload a backup file instead,
+and Steward comes up with its users, settings and key. There is nothing to
+keep yet, so no safety backup is made.
+
+The backup's secret key is put in place as `/data/secret.key`. If
+`STEWARD_SECRET_KEY` is set instead, it must equal the backup's key, or the
+restore stops before anything changes.
+
+### From the command line
+
+`steward restore` writes into an **empty** database. Pass `--wipe` to drop
+everything in the database's public schema first. Use it when Steward
+can't start at all.
+
+`steward restore --in-place` does what Settings → Backups does for a backup
+of the same schema, from the shell: it replaces what the backup carries and
+keeps the audit log and history, in one transaction. Restart Steward after
+it so the restored settings load.
+
+1. Stop Steward (scale its service to 0) so nothing writes during the restore.
+2. Download the backup file from Archives.
+3. Run the restore with the image, pointed at the database:
+
+   ```sh
+   docker run --rm \
+     -e STEWARD_DATABASE_URL=postgresql+asyncpg://steward:…@db:5432/steward \
+     -v steward_data:/data -v "$PWD":/restore \
+     git.fabledsword.com/bvandeusen/steward:latest \
+     steward restore /restore/steward-20261012T030000Z.tar
+   ```
+
+   - The backup's secret key is written to `/data/secret.key`.
+   - If `STEWARD_SECRET_KEY` is set instead, it must equal the backup's key,
+     or the restore stops and says so. Read the key with
+     `tar -xOf <file> secret.key`.
+4. Start Steward. On start it runs any migrations the backup is behind on.
+
+**How the restore works.** pg_restore runs in its three sections: schema,
+then data, then constraints and indexes. Between the data and the
+constraints, a kept row that points at a left-out row is put right by its
+foreign key's ON DELETE:
+- a SET NULL link is cleared and the row kept;
+- a row whose foreign key would cascade or block is removed.
+
+pg_restore 17 writes each section as a script for psql. It runs against
+Postgres 16 with one setting line dropped (`transaction_timeout`, which 16
+doesn't know).
+
+CI checks the whole path on every build. The built image backs up its own
+database and checks the backup against it. It changes a setting and puts
+the backup back in place: the setting returns and the audit log stays.
+Then it restores into a second, empty database, checks a setting came back
+and the metrics table and audit log are empty, and starts Steward on it.

docs/core/mcp.md

@@ -52,7 +52,7 @@ authentication. The AI access page shows the recipes for both.
 | Bearer tokens only, no OAuth | `steward/core/api_tokens.py`. Only a hash is stored, and the token is shown once. |
 | A token never acts above its user | Effective role = min(token role, user's **current** role). Lowering a user lowers every token they hold; disabling the user stops them all. |
 | Default deny | A tool is reachable only if it was registered with a role (`register_tool(fn, role=..., read_only=...)`). `tools/list` shows a caller only the tools their role allows. A call above the role is refused, and the refusal names the role it needs. |
-| Plugin tools follow the plugin | A disabled plugin's tools are hidden and refused. |
+| Plugin tools follow the plugin | A disabled plugin's tools are hidden and refused. The host agent is built in and always on (it can't be disabled), so its tools are always there. |
 | Dry run first | `run_playbook`, `create_schedule` and `set_alert_action` default to Ansible check mode (`--check --diff`). Changes are applied only when the caller passes `check=false`. A playbook its author flagged `# steward:confirm: true` also needs `confirmed=true`. `prune_docker` previews unless `check=false`. |
 | Secrets stay put | Playbook variables Steward detects as secret reach the run but are never stored, audited or returned. Schedules and alert actions refuse them, since they would have to store them. Inventory variables are listed by name only, and HTTP monitor header values are redacted. |
 | Every change is audited | Each write records an audit event as `<user> via MCP`, with the token id in its detail. `list_audit_events(user="<user>")` includes those events. |
@@ -75,23 +75,59 @@ Each role includes the tools of the roles above it in this list.
 | Hosts and metrics | `list_hosts`, `get_host`, `get_host_metrics`, `query_metrics` |
 | Monitors | `list_monitors`, `get_monitor` |
 | Alerts | `list_alerts`, `get_alert_history`, `list_alert_rules`, `get_alert_rule_options`, `list_maintenance_windows` |
+| Metric anomalies | `list_anomalies` (metrics that left their normal for the hour, to review first), `get_anomaly` (one anomaly with each occurrence against what is usual at that hour, the metric around it, what else happened on its host — alerts, Ansible runs, container deaths and starts, maintenance, reboots, other anomalies — and how earlier ones on the same metric were judged), `get_metric_normal` (the learned normal slot by slot, and the level that would be flagged), `get_anomaly_settings` (the review window and each floor) |
+| Thresholds | `get_thresholds` (the warn and crit cutoffs CPU, memory, disk, load and temperature are graded by on the pages) |
 | Automation | `list_playbooks`, `get_playbook`, `list_inventory`, `list_ansible_runs`, `get_ansible_run`, `wait_for_run`, `list_schedules` |
 | Audit | `list_audit_events` |
+| Timeline | `get_timeline` (everything that happened, newest first, in one list: alerts firing and clearing, playbook runs and what they changed, maintenance, reboots, deploys and crash-loops, reviewed anomalies, changes recorded by hand, changes to Steward's settings; by host, area or kind, and kept after the records it came from age out) |
+| Tickets | `list_tickets` (known problems someone recorded, open first; by host, subject or area), `get_ticket` (one ticket: its description, the alert rules and anomalies it links, its resolution and its log). `list_alerts` and `list_anomalies` mark what an open ticket covers as `known_issue`; `get_overview`, `get_host`, `get_monitor` and `get_container` carry `open_tickets` |
 | Docker plugin | `list_containers`, `get_container`, `list_swarm_services`, `search_container_logs`, `get_docker_disk_usage` |
-| Host agent plugin | `list_agents` |
+| Host agent (built in, always on) | `list_agents`, `list_maintenance_plans` (each plan's group, weekly slot, next run and latest run: how far it got, who it skipped and why), `get_maintenance_job` (a host's update jobs or one job: state, packages updated, whether it rebooted, why it failed), `get_host_senders` (who sent or received a host's network traffic over a window: its containers and NFS mounts ranked at the host's peak, and what none of them accounts for) |
+| Traefik plugin | `list_routers` (each router graded as the Traefik page grades it, with the status codes behind a problem, what they mean and the swarm service behind it), `get_router` (one router's history, codes, normal and anomalies), `list_certificates` (served certificates, days left, the routers they cover), `get_traffic` (the access log's clients, countries and routers, and Traefik's own figures) |
+| UniFi plugin | `get_network` (the WAN, clients, Wi-Fi networks, speed tests and devices by status), `list_devices`, `list_clients` (everyone seen, or the newest poll's top talkers), `list_network_events`, `list_alarms`, `get_network_config` (networks and VLANs, port forwards, firewall rules) |
+| SNMP plugin | `list_snmp_devices` (each device's state, newest readings in their units and the readings it didn't return), `get_snmp_device` (one device's readings over a range). Never the community or v3 keys |
+| Syslog plugin | `list_syslog_sources` (the devices sending syslog: added, offered or ignored, each with its rate, last heard and newest line), `search_syslog` (an added source's kept lines, or a host's, filtered by text, severity and worse, app and time, newest first), `list_syslog_patterns` (the phrases and severities counted, the metric each writes and the ready-made ones not added), `add_syslog_preset` (admin: add a ready-made one) |
+| Prometheus plugin | `list_prometheus_targets` (each target's state and its newest scrape: series kept, dropped and over the cap), `get_prometheus_target` (a target's stored series by metric, with labels and newest values). Never the token, and a password in a URL is masked |
 
 ### Operator: run playbooks, change monitoring
 
 | Area | Tools |
 |---|---|
-| Automation | `run_playbook`, `cancel_ansible_run`, `create_schedule`, `update_schedule`, `delete_schedule` |
+| Automation | `run_playbook`, `cancel_ansible_run`, `create_schedule`, `update_schedule`, `delete_schedule`, `update_group_members` (put hosts into an inventory group or take them out; a group alert rule follows within a minute) |
 | Monitors | `create_monitor`, `update_monitor`, `set_monitor_enabled` |
-| Alerts | `create_alert_rule`, `update_alert_rule`, `acknowledge_alert`, `create_maintenance_window`, `end_maintenance_window` |
+| Alerts | `create_alert_rule`, `update_alert_rule`, `acknowledge_alert`, `send_test_alert`, `create_maintenance_window`, `end_maintenance_window` |
+| Metric anomalies | `review_anomaly` (mark one or several normal, so their hours are learned, or real, with a note saying why) |
+| Timeline | `log_change` (record a change Steward can't see happen — a config edited by hand, a disk swapped — on the hosts it was done on, under an area; shown as by "<you> via MCP"), `update_change` and `delete_change` (correct or take back a recorded change: its author or an admin) |
+| Tickets | `open_ticket` (a known problem, on its hosts and what it is about, linked to alert rules and anomalies; goes on the timeline), `update_ticket` (its fields and links), `comment_ticket`, `resolve_ticket` (with what fixed it, which goes on the timeline), `reopen_ticket` |
 
 ### Admin: destructive and remediation
 
-`set_alert_action` (the playbook a rule runs when it fires), `delete_monitor`,
+`set_anomaly_settings` (the anomaly review window and each normal's floor, as
-`delete_alert_rule`, `prune_docker`.
+Settings → Thresholds sets them), `set_thresholds` (the warn and crit cutoffs host
+readings are graded by, as the same page sets them), `set_alert_action` (the playbook a rule runs when it fires), `delete_monitor`,
+`delete_alert_rule`, `prune_docker`, `run_maintenance` (update a host now, rebooting
+only if the update needs it, as the host page's Update now does).
+
+`save_playbook` and `delete_playbook` write this instance's own playbooks into
+`steward-local`, the same as Steward's editor (#5748). That is automation for
+these hosts that isn't part of Steward itself; Steward's built-in playbooks are
+changed in its repository, and a git source by its repo, so both are refused.
+A save must parse as a list of plays or nothing is written; it won't overwrite
+an existing playbook without `replace=true`; and it then runs
+`ansible-playbook --syntax-check` (no host contacted) and says what that found.
+The loop for building one is save → `run_playbook` in check mode on one target
+→ `get_ansible_run` → fix and save again → apply with `check=false`. The audit
+row records the size, never the content.
+
+`list_backups`, `check_backup` and `make_backup` are Settings → Backups for an
+agent (#5836), admin only like the page, since a backup holds every secret
+Steward keeps. `list_backups` gives the schedule, the next run and the last 30
+runs with the check each one got after it was sent (`bucket=true` also lists the
+bucket, which an append-only key may refuse). `check_backup` fetches one and
+compares it with what is in place, changing nothing. `make_backup` starts one in
+the background and is audited. Restoring stays in the browser, behind its typed
+confirmation. `get_overview` reports, for every role, a scheduled backup that
+failed, is overdue, or has no bucket set up.
 
 The client sees each tool's full contract: arguments, what an empty value
 means, and what comes back. That contract is the tool's docstring, so read the
@@ -126,7 +162,12 @@ Every tool follows these conventions (`steward/mcp/tools/_shape.py`):
 
 Core tools: add an `async def` to a module in `steward/mcp/tools/` and list it
 in that module's `register()` with its minimum role and `read_only`. A plugin
-exposes `register_mcp_tools(register)`, and optionally `mcp_overview(db)` to
+exposes `register_mcp_tools(register)`, optionally `mcp_overview(db)` to
-contribute findings to `get_overview`. See
+contribute findings to `get_overview`, and optionally
+`mcp_events(db, host_id, host_name, since, until)`: what it knows happened on a
+host in a window, as a list of `steward.mcp.tools._events.event(...)`, which
+`get_anomaly` shows beside an anomaly; for a resource with no host, optionally
+`mcp_resource_events(db, source, resource, since, until)`. Core adds the
+stretches Steward itself wasn't recording to every anomaly. See
 [Writing a plugin](../plugins/writing-a-plugin.md). A tool that changes
 anything calls `audit_mcp(...)` from `steward/mcp/tools/_audit.py`.

docs/core/tickets.md

@@ -0,0 +1,55 @@
+# Tickets
+
+A ticket records a known problem that takes more than a moment to fix: a disk failing slowly, a service that keeps restarting, a backup that times out at night. It says what is wrong, what has been tried and, once resolved, what fixed it. Acknowledging an alert only quiets it. A ticket keeps the story, and while it is open the alerts and anomalies it covers read as a **known issue**.
+
+Open the list from **Tickets** in the sidebar. A host's page, a monitor's page and a container's page each show a **Known issues** card with their open tickets. MCP clients use `list_tickets`, `get_ticket`, `open_ticket`, `update_ticket`, `comment_ticket`, `resolve_ticket` and `reopen_ticket`.
+
+**Source:** `steward/tickets/` (`service.py` for every write, `routes.py` for the pages), `steward/mcp/tools/tickets.py`
+**Tables:** `tickets`, `ticket_links`, `ticket_notes` (`steward/models/tickets.py`)
+
+---
+
+## What a ticket holds
+
+| Field | |
+|---|---|
+| Number | `#12`, in order of opening |
+| Title | What is wrong, one line (up to 200 characters) |
+| Area | One of the timeline's areas (Settings → Timeline) |
+| Hosts | The hosts it is on, by name; none for a problem not on any one host |
+| Subject | What it is about when that is not a host as a whole: an alert rule's resource (`web-1:/`), a container (`web-1/nginx`), a monitor's name |
+| Description | What is seen, since when, what has been tried |
+| Links | Alert rules and metric anomalies it covers |
+| Log | Opened, comments, edits (which fields changed, what was linked or unlinked), resolved, reopened, each with who and when |
+
+A ticket is **open** or **resolved**. Resolving one needs a resolution: what fixed it, or why it no longer matters. Reopening clears it, with an optional reason.
+
+## Opening one
+
+- **From an alert:** **Open a ticket** on an active alert (Alerts page) fills in the title, area, hosts and subject from the rule, and links the rule.
+- **From an anomaly:** **Open a ticket** on an anomaly to review, or one reviewed as real, does the same and links the anomaly.
+- **From a page:** the **Known issues** card on a host, monitor or container opens one about it.
+- **From scratch:** **Open a ticket** on the Tickets page.
+- **Through MCP:** `open_ticket(title, area, description, hosts, subject, alert_rules, anomalies)`.
+
+Opening, editing, commenting, resolving and reopening need an operator or admin. Viewers read everything.
+
+## Known issues
+
+An alert or anomaly is a known issue while an open ticket:
+
+- links its alert rule (or the covering group rule it came from), or the anomaly;
+- names its resource as its subject; or
+- is about its host as a whole (no subject), when the resource is the host itself.
+
+The Alerts page and the anomaly lists show a **Known issue** badge with the ticket. `list_alerts` and `list_anomalies` return the same as `known_issue`. `get_overview` lists up to ten open tickets in `open_tickets` and counts them all in `counts.tickets_open`. `get_host`, `get_monitor` and `get_container` carry their own `open_tickets`.
+
+## On the timeline
+
+Opening, resolving and reopening a ticket each write a timeline entry under the ticket's area, on its hosts and subject (`ticket.opened`, `ticket.resolved`, `ticket.reopened`, kind Ticket). The resolution is the resolved entry's note, so the timeline keeps what fixed it. Comments and edits stay in the ticket's own log.
+
+Every step is audited (`ticket.opened`, `ticket.edited`, `ticket.commented`, `ticket.resolved`, `ticket.reopened`). Those audit rows stay off the timeline, which holds the ticket's own entries.
+
+## Backups
+
+Tickets are not in Steward's backups yet, the same as the timeline itself (`steward/backup/contents.py`, `EXCLUDED`).

docs/core/timeline.md

@@ -0,0 +1,66 @@
+# Timeline
+
+The timeline is one list of everything that happened to the infrastructure, newest first: alerts firing and clearing, playbook runs and what they changed, maintenance runs and reboots, deploys and crash-loops, anomalies once reviewed, changes recorded by hand, and changes to Steward's own settings. Open it from **Timeline** in the sidebar, from a host's **History** card, or through the `get_timeline` MCP tool.
+
+Each entry stands on its own. An alert event, an Ansible run, a maintenance job or an audit row is pruned after its own retention, but the entry for it stays. Its link then reads "Details no longer kept".
+
+**Source:** `steward/timeline/`
+**Table:** `timeline_entries` (`steward/models/timeline.py`)
+
+---
+
+## What is recorded
+
+Every source writes its entry in the same transaction as its own record, through `steward/timeline/record.py`:
+
+| Source | Written by | Entry types |
+|---|---|---|
+| Alert events | `core/alerts._add_event` | `alert.fired`, `alert.resolved`, `alert.acknowledged`, `alert.silent`, `alert.resumed` |
+| Ansible runs, when they finish | `ansible/executor.start_run`, and the startup sweep of interrupted runs | `ansible.changed`, `ansible.unchanged`, `ansible.failed`, `ansible.check` |
+| Audit rows | `core/audit.log_audit` | `steward.config`, `steward.access`, `steward.signin`, `steward.backup` |
+| Metric anomalies: one entry each, whose type follows its review | `core/anomalies` (opened, ended, marked, lapsed) | `anomaly.to_review`, `anomaly.real`, `anomaly.normal`, `anomaly.lapsed` |
+| Maintenance jobs, when they end (and again if the host comes back late) | host agent: `maintenance`, `cadence` | `maintenance.done`, `maintenance.failed` |
+| A host away: a reboot, a crash, lost contact, the agent restarting; kept up to date until it ends | host agent: `downtime.record_events` | `host.rebooted`, `host.lost_contact`, `host.agent_restarted` |
+| Container snapshots: a new image, a recreated container, a restart or death | Docker: `ingest.persist_host_docker` → `timeline.from_snapshot` | `docker.deployed`, `docker.crash_loop`, `docker.restarted` |
+| A change recorded by hand, from the Timeline page, a host's History card or MCP `log_change` | `timeline/changes.py` | `change.recorded` |
+| Tickets: opened, resolved (the resolution is the entry's note) and reopened | `tickets/service.py` | `ticket.opened`, `ticket.resolved`, `ticket.reopened` |
+
+A source whose record changes after it is written keeps one entry for it (`record.record_or_update`): a downtime gets its length when it ends, an anomaly changes type when it is reviewed.
+
+Docker's units are swarm services where a container has one, so a service's tasks, replaced on every deploy and crash, read as one subject. A swarm task moving to another host is not a deploy. A crash loop uses the same rule as the Docker pages: Settings → Thresholds' deaths within an hour. The death that reaches it opens one `docker.crash_loop` entry, and each later one extends it until the container has been quiet for an hour. The deaths before it stay as hidden `docker.restarted` entries.
+
+Some audit actions are left to another writer, because the event itself is recorded there: starting a run, acknowledging an alert, starting maintenance, and recording a change by hand. A person's own preferences (time zone, email settings) are not recorded. `catalog.audit_type` holds that mapping.
+
+On its first start, Steward fills the timeline from the last 90 days of records that predate it (`service.backfill`): core's alert events, finished runs and audit rows, and each loaded plugin's own through its `timeline_backfill(session, since)` (the host agent's maintenance jobs and downtime). It fills from each source once and records which in `timeline.filled`, so a plugin loaded later fills on the next daily turn. An entry for a record that is already covered is skipped. Docker has no fill: its events can't tell a deploy from a restart after the fact.
+
+`get_anomaly`'s "what else was going on" reads the timeline for the host or resource, hidden types included, beside each occurrence.
+
+## Recording a change by hand
+
+Steward can't see every change happen: a config edited on a host, a disk swapped, a service moved. Whoever made it records it, with **Record a change** on the Timeline page or a host's History card, or the `log_change` MCP tool. It says what was done, the area it is about, when, the hosts it was done on (none for a change not on any one host), optionally what it was done to (a container, a file) and a longer detail.
+
+An operator or admin can record one. Its author can edit or delete it from its **Edit** link, or with the `update_change` and `delete_change` MCP tools, and so can an admin. One recorded through MCP is shown as by "<user> via MCP" and is editable by that token's user. Recording, editing and deleting are audited (`timeline.change_recorded`, `…_edited`, `…_deleted`), and those audit rows stay off the timeline, which holds the change itself.
+
+## Areas and kinds
+
+An entry has a **kind**, which says how it came about: Change, Automation, Alert, Anomaly, Ticket, Observed (Steward saw it happen with no rule or person behind it: a reboot, a crash-loop) or Steward. Kinds are fixed in `catalog.KINDS`.
+
+An entry also has an **area**, which says what part of the infrastructure it is about. The areas start as Service, Host OS, Hardware, Storage, Network, Security and Monitoring.
+
+Each entry is written with an area of its own. An alert's area comes from what its rule watches (`catalog.area_of_metric`): drive health is Hardware, a mount is Storage, an HTTP monitor is Service. A change recorded by hand has the area its author picked. Other entries use their type's area.
+
+## Settings → Timeline
+
+| Setting | Key | Default |
+|---|---|---|
+| Areas, in order: rename, add, or remove one and move its entries to another | `timeline.areas`, `timeline.area_moved` | the seven above |
+| Each type's area ("As recorded" keeps each entry's own) and whether it is shown | `timeline.types` | from the catalog |
+| Keep for, in days (0 keeps everything) | `timeline.retention_days` | 0 |
+
+These are applied when the timeline is read, never written onto an entry. A change re-sorts the whole history. A hidden type is still recorded, so showing it again brings its entries back.
+
+Hidden by default: dry runs, playbook runs that changed nothing, sign-ins, single container restarts, the agent restarting, and anomalies not yet reviewed, accepted as normal or lapsed.
+
+## Adding a type
+
+A plugin registers its types in `setup()` with `catalog.register_type(key, label, kind, area, shown, about)`, and the records its entries point to with `catalog.register_ref(ref_type, model)`, so an entry whose record has aged out reads "Details no longer kept". It writes entries with `record.record(session, type=…, at=…, summary=…, hosts=…, ref=(ref_type, id))`, or `record.record_or_update` for a record that changes. It may add `timeline_backfill(session, since)` to its `__init__.py` for records that predate the timeline. Settings → Timeline lists the new type at once. `plugins/host_agent/timeline.py` and `plugins/docker/timeline.py` are worked examples.

docs/plugins/homeassistant.md

@@ -0,0 +1,54 @@
+# Home Assistant plugin
+
+Keeps a Home Assistant instance running: shows whether backups are happening and whether the system has a problem, and installs Core, OS and add-on updates, on a click or a weekly schedule, each only after a fresh backup is confirmed. It looks after Home Assistant itself and doesn't read your devices. Each update goes on the timeline. Nothing is added to Home Assistant: you sign in, and Steward uses its standard API.
+
+## Setting it up
+
+1. In Home Assistant, turn on automatic backups (Settings → System → Backups).
+2. Make a user for Steward (Settings → People → Add person), as an administrator if Steward should install updates.
+3. On Steward's Home Assistant page, choose Connect, enter Home Assistant's address, and continue to its sign-in page. Sign in as that user.
+
+Your browser and Steward both need to reach Home Assistant at that address. Home Assistant sends your browser back to Steward's Public base URL (Settings → General), so open Steward at that address.
+
+## What Steward holds
+
+**Steward stores a credential with the same access as the user who signed in.** Home Assistant has no narrower permission, so an administrator's credential can do anything that administrator can, including controlling every device. Steward only uses it to read status, take backups and install updates. It is stored encrypted.
+
+We recommend a service account, a user made just for Steward, rather than your own. What Steward does then appears under its own name in Home Assistant's logs, so it's easy to tell apart from what people do.
+
+Installing needs an administrator, because Home Assistant only lets administrators take backups. With any other user, Steward shows status only. Steward learns which it has by asking once after sign-in, and a user who isn't an administrator is refused that one request, which Home Assistant may show as a failed login notification.
+
+Removing the connection ends the sign-in on Home Assistant. You can also end it yourself under Security on that user's profile, and Steward then stops asking until you sign in again. Home Assistant bans an address after a few refused requests, so Steward stops after one refusal until you sign in again or choose Try again.
+
+## Installing an update
+
+Install asks once, then Steward:
+
+1. Asks for a backup and waits until Home Assistant's last successful backup time moves. If it doesn't, nothing is installed.
+2. Asks for the update and follows it until the new version is installed, or until a deadline: 20 minutes for an add-on, 30 for Core, 45 for the OS.
+
+One runs at a time, and it carries on if Steward restarts. Firmware and custom integrations are left to Home Assistant, and Supervisor updates itself.
+
+An OS update takes effect only after the machine reboots, and Install leaves that to you. Steward records the update as done once Home Assistant reports the new version, which it does before the reboot. Home Assistant then raises a repair asking for the reboot, which appears under System problems until you do it.
+
+## System problems
+
+Each read also asks Home Assistant whether the system itself has a problem, and the page and the overview show what it says:
+
+- open repairs, from Settings → System → Repairs
+- the Supervisor calling the system unhealthy or unsupported
+- less than 10% of the disk free
+- starting in recovery mode, when its configuration didn't load
+- an add-on set to start on its own that isn't running
+
+Repairs need an administrator's sign-in. Disk space and add-ons come from the Supervisor, so an install without one doesn't report them, and the page says so.
+
+## Scheduled updates
+
+Set a day and time under Scheduled updates on the page. It's off until you do. At that time Steward installs what's waiting, add-ons first, then Core, then the OS, one at a time and each after its own confirmed backup. It starts nothing new more than 4 hours after the slot. The first failure ends the run and the rest wait, and the page, the overview and `update_failed` say so. A Core update restarts Home Assistant with nobody there to watch, and an update can change how automations behave.
+
+An OS update installed by the schedule also reboots the machine, with nobody there to watch. It counts as done once the new version is running. If that hasn't happened 20 minutes after the reboot, for example because the machine didn't come back or fell back to the old version, the update is marked failed.
+
+## Metrics
+
+Under source `homeassistant`, resource `homeassistant`, read every 5 minutes: `is_up`, `updates_waiting`, `backup_age_hours`, `problems_open`, `disk_free_pct` and `update_failed`. Useful alert rules: `backup_age_hours` above 26 for a missed daily backup, `problems_open` above 0, and `update_failed` above 0. A rule's no-data warning covers Home Assistant going quiet.

docs/plugins/host-agent-design.md

@@ -58,9 +58,13 @@ Give Steward a fleet-glance view of remote Linux hosts — current resource usag
 ### File layout on the target host
 
 ```
-/usr/local/lib/steward-agent/agent.py       # the script, target ~300 lines
+/usr/local/lib/steward-agent/agent.py       # the agent (runs as steward-agent)
+/usr/local/lib/steward-agent/maint.py       # the maintenance helper (runs as root, #5766)
 /etc/steward-agent.conf                      # key=value config, 0640 root:steward-agent
-/etc/systemd/system/steward-agent.service    # unit file
+/etc/systemd/system/steward-agent.service    # the agent's unit
+/etc/systemd/system/steward-maint.{path,service}  # starts the helper on a request
+/var/lib/steward-agent/requests/             # the one place the agent writes (0700 steward-agent)
+/var/lib/steward-maint/results/              # the helper's outcomes (root, readable)
                                                 # dedicated system user: steward-agent
 ```
 
@@ -113,9 +117,291 @@ Python 3.8+ stdlib only. `urllib.request` for the POST, plus `json`, `os`, `shut
 ### Signals and lifecycle
 
 - **SIGHUP** → re-read config file without restart (lets admins change interval/mounts cleanly).
-- **SIGTERM** (from systemd) → finish current poll cycle → flush buffer one last time → exit.
+- **SIGTERM** (from systemd) → finish current poll cycle → flush buffer one last time, with a `stopping` event saying whether the host is going down too (`systemctl is-system-running` says `stopping`) → exit.
 - **Systemd restart policy** → `Restart=on-failure`, `RestartSec=10`.
 
+### Mounts that don't answer (1.14.0, #5760)
+
+Each mount is read in a daemon thread under one 5 s deadline. A hard-mounted NFS share whose server stops blocks `statvfs` in the kernel with no timeout; read inline, that froze the whole agent. A mount that misses the deadline is sent as `{"mount", "stalled": true}`, and isn't probed again while its last probe is still blocked. Steward counts them as `mounts_stalled`.
+
+### NFS delay per mount (1.22.0, #5996)
+
+`mounts_stalled` says a mount is stuck; it can't say a share was slow before it stuck, or how slow it was when it came back. Each sample also reads `/proc/self/mountstats`, the kernel's own cumulative count of every NFS request each mount has finished. Reading it never touches the share, so it answers while one is stuck. The agent sends each NFS mount's requests since the last sample as `nfs: [{mount, ops_per_sec, retrans, timeouts, rtt_ms, exe_ms, read_ms, write_ms}]`: `rtt_ms` is the network and server's round trip, `exe_ms` that plus the time queued on the host, which is what a program waits. The delays are averages over the requests that finished, and are left out when none did. A mount new since the last sample, or whose counters went back (remounted), waits for the next one.
+
+**Bytes per mount (1.25.0, #6115).** Each op's line also counts the bytes sent and received on the wire, so each mount's row carries `tx_bps` and `rx_bps` too: the share's part of the host's network traffic, which belongs to no container. A large copy to the NAS shows here rather than as traffic nobody sent.
+
+Steward stores them per mount (`nfs_*` on `<host>:<mount>`), and the slowest mount's `exe_ms` on the host as `nfs_exe_max_ms`, for a rule ("NFS slow": over 1000 ms for 3 checks) and for the host page's NFS chart beside `mounts_stalled`. Each mount's bar on the host page says what a request took and how many were sent again or timed out. A request is counted when it finishes, so a stuck share shows nothing until it comes back, then its long requests all at once.
+
+### A container's memory is its processes' own (1.23.0, #6030)
+
+Each running container sends `mem_used_bytes` and `mem_used_pct` (of its limit, or of the host's RAM when it has none) and, beside them, `mem_cache_bytes`. Used is anon plus shmem from the container's cgroup memory stats (v1: rss plus shmem). It is what the processes hold and can't give back without swap, and it counts Postgres's shared buffers, which live in shmem. Cache is the file cache charged to the container (v2's `file`, v1's `cache`, shmem taken out). The kernel keeps what the container read or wrote for as long as nothing else needs the memory, and gives it back when something does.
+
+Before this, the agent sent Docker's working set (usage less `inactive_file`) as `mem_usage_bytes`. That figure still counts recently used file cache. On 2026-10-17 a file-heavy container with no memory limit read 191 GB that way, while its processes held 2.2 GB. The host's own figure (`MemTotal − MemAvailable`) already left cache out, so the two disagreed. The new names keep the two meanings apart in history and in learned normals.
+
+### Every Docker read is bounded (1.15.0, #5764)
+
+One Docker answer must arrive within 15 s and fit in 32 MiB, and one log answer within 5 s and 4 MiB. Past either limit the read is cut off, not waited on. A per-read timeout alone never ends a body that keeps trickling in. Container logs are read since each container's cursor, but only its newest 2,000 lines, all containers within 10 s. A container that writes more than that gets a `_steward` note saying older lines were skipped. A container whose answer is too big skips ahead to now. Before this, a 1.13.0 agent re-read a growing backlog every interval and went silent for a day at 2.1 GB. Each sample also carries the agent's own `agent` footprint (`rss_bytes`, `cpu_pct`), which Steward records as `agent_rss_bytes` and `agent_cpu_pct`.
+
+### What the host has waiting (1.16.0, #5765)
+
+Once an hour, and when the agent starts, a background thread asks the package manager what it would upgrade, so a slow apt never holds up a sample. On apt hosts it uses `/usr/lib/update-notifier/apt-check` (pending;security), or `apt list --upgradable` when that's missing. On dnf hosts it uses `dnf -C check-update`, `dnf -C updateinfo list --security`, and `needs-restarting -r` for whether to reboot; yum hosts the same through `yum -C` (1.21.0, #5803). On zypper hosts it uses `zypper --no-refresh list-updates` (`--dup` on Tumbleweed, which upgrades by distribution), `list-patches --category security` for the needed security patches (Tumbleweed has none to list), and `zypper needs-rebooting`. On pacman hosts it uses `checkupdates` from pacman-contrib, which install-deps.sh installs; pacman says nothing of security, and a reboot is required once the running kernel's `/usr/lib/modules/<release>` is gone, i.e. the kernel was upgraded under it. None of this needs root, and apart from `checkupdates` none of it needs the network: the commands read lists the system's own timers already fetched. `checkupdates` syncs a copy of the package lists into its own temporary directory, so it reaches the host's mirrors once an hour but never touches pacman's real database. Each command has 120 s. The last answer rides every sample as `updates`. Every sample also reads Debian's `/var/run/reboot-required` (and `.pkgs`) as `reboot`. Steward records `updates_pending`, `updates_security` and `reboot_required` (1/0) for rules, and keeps the newest on the registration. The host page's Agent card leads with "Reboot required since …", then "N updates waiting (M security)" or "Up to date", with when the check ran. `state["updates_due"] = True` asks for a check on the next sample; a maintenance job uses it once it has updated the host.
+
+
+### A root helper for updates and reboots (1.17.0, #5766)
+
+The agent can't update or reboot its host: it runs as `steward-agent` under
+`NoNewPrivileges` and `ProtectSystem=strict`, so no sudo is possible, and that
+is how it should stay. A separate helper, `maint.py`, runs as root, started by
+systemd when the agent leaves it a request:
+
+- The agent writes `<id>.json` = `{"id", "action", ["node"]}` into
+  `/var/lib/steward-agent/requests`, the one path its unit lets it write
+  (`ReadWritePaths=-…`; the `-` keeps an old host without the directory
+  starting). It writes a dot-file first and renames it, so the helper never
+  sees half a request.
+- `steward-maint.path` (`PathExistsGlob=…/requests/*.json`) starts
+  `steward-maint.service`, a oneshot running `maint.py` as root.
+- The helper opens each request with `O_NOFOLLOW`, wants a regular file under
+  4 KiB, and accepts only `check`, `update`, `reboot`, `drain` or `undrain`, an id of
+  letters, digits and dashes, and a node id in the swarm's own 25-character
+  form. It deletes every request, valid or not, and writes the outcome to
+  `/var/lib/steward-maint/results/<id>.json` (root's; kept a week). A request
+  whose id already has a result is not run again.
+- **check** (1.24.0, #6098): what the update would upgrade, changing nothing
+  but the package lists: `apt-get update` then `apt-get -s dist-upgrade`;
+  `dnf`/`yum check-update`; `zypper list-updates` (`dup --dry-run` on
+  Tumbleweed); `pacman -Sy` then `-Qu`. It says whether Docker is among it
+  (`docker`, `docker-ce`, `docker.io`, `docker-engine`, `moby-engine`,
+  `containerd`, `containerd.io`: the packages whose install scripts restart
+  the daemon), and lists the running containers that aren't swarm tasks,
+  each with its restart policy (at most 40).
+- **update**: `apt-get update` then `dist-upgrade` keeping existing config
+  files, with `NEEDRESTART_MODE=l` so needrestart only lists services rather
+  than restarting them. A package's own install script still restarts its
+  service: Docker's does, stopping every container on the host, which is
+  what the check is for;
+  `dnf -y upgrade --refresh` (or yum's); `zypper refresh` then
+  `zypper update` (`zypper dup` on Tumbleweed), counting zypper's 102/103
+  "reboot / restart needed" exits as success; or `pacman -Syu`
+  (#5804). The result says how many packages changed and whether a reboot is
+  now required.
+- **reboot**: writes `rebooting`, syncs, `systemctl reboot`. Any request after
+  it waits for the next boot.
+- **drain / undrain**: `docker node update --availability drain|active`, on
+  the given node or this host's own. Only a manager can; a worker's helper says
+  so in its result.
+
+The agent reports each result's new state once, as a `maintenance` event with
+its next post that gets through (results older than a day are not news). When
+an update finishes it re-checks what's waiting straight away rather than at
+the next hourly check. Its metadata says `maint_helper: true` once the path
+unit is installed and it can write its request directory; the host's Agent
+panel shows that under Manage agent. **Update agent** installs the helper on a
+host that has an older agent.
+
+So a stolen agent token can make its host update, reboot or drain, and nothing
+else. Steward handing a host a job and following it through the reboot is
+#5767 (below); the cadence that does it one host at a time is #5768.
+
+### A maintenance job, followed through the reboot (1.18.0, #5767)
+
+Steward keeps one `host_maintenance_jobs` row per job and moves it on only from
+what the agents post, so a restart of Steward, or a reboot of the host Steward
+runs on, loses nothing. `plugins/host_agent/maintenance.py` has the states and
+transitions (model-free, unit-tested) and the database side.
+
+```
+queued → [checking →] [draining → settling →] updating
+       → [draining → settling →] [rebooting →] [undraining →] [recovering →] done
+any state → failed (a step's result said so) | timed_out (its deadline passed)
+draining / settling / updating / rebooting, once drained → undraining → failed | timed_out
+```
+
+**A job never ends leaving its drain behind (#6387).** On 2026-10-25 a job
+drained VDNT-Docker02, which ran Traefik and Steward. Its agent's result
+reaches Steward through Traefik, and Traefik took 11.5 minutes to start on
+Docker03 (its image had to be pulled). The job gave up at the 10-minute drain
+deadline and finished `timed_out` where it stood, leaving the node drained
+for hours. Now a job that fails or times out while its node may be drained
+(the drain was handed out, or went through and wasn't undone) goes to
+**undraining** first, with the outcome kept in `ends_as`, and finishes as that
+once the node is active again. If the undrain goes wrong too, `detail` says
+both. The drain deadline is 30 minutes, so a result held up by the proxy
+moving still lands.
+
+**Docker upgraded, drained first (1.24.0, #6098).** On 2026-10-18 an update
+on VDNT-Docker02 restarted the Docker daemon six seconds in, before the drain
+that came only with the reboot: every container on the node stopped, and
+portal_app was down until it was redeployed by hand. A job for an agent of
+1.24.0 or later now starts with a **check**. When the update upgrades Docker,
+a swarm node is drained before it; otherwise it is drained only before a
+reboot, and a node is drained at most once a job. After a drain, **settling**
+gives the node's tasks 5 minutes (`DRAIN_SETTLE`) to move before the update
+or reboot goes ahead. The containers the check found running outside the
+swarm are kept on the job (`checked`); once the work is over, **recovering**
+waits up to 10 minutes (`RECOVER_LIMIT`) for each to be seen running again
+in the Docker plugin's stored containers, and fails the job naming any that
+aren't, marking those with no restart policy. settling and recovering are
+Steward's own waits, moved on by the minute tick. An older agent gets no
+check, and its job runs as before.
+
+- **Hand-off.** Steward's answer to an ingest post carries
+  `"maintenance": [{"id", "action", ["node"]}]`, the step that host's agent
+  should carry out now. It is repeated on every post until a result for that
+  step id arrives; the agent leaves each id for the helper once
+  (`take_maintenance`), and the helper refuses an id it already has a result
+  for, so a repeat, or a re-hand after the agent restarts, never runs twice.
+  A step id is `<job id>-<action>`.
+- **Results** are the `maintenance` events of #5766, matched to the job by step
+  id, so a drain reported by a manager reaches the job of the node it drained.
+- **The reboot completes** when the host's agent sends `started` with a
+  `boot_at` after the reboot was handed out (60 s of clock slack). An agent
+  restart on a host that stayed up has an old `boot_at` and changes nothing.
+- **Reboot only when required**: the update's result says whether a reboot is
+  needed; a job made with "never reboot" stops after the update.
+- **Swarm nodes** are drained first and made active after (see above for
+  when). The node is found
+  by hostname in the Docker plugin's swarm nodes; a manager drains itself, a
+  worker is drained by a manager whose agent has the helper. With no such
+  manager the job is refused rather than rebooting an undrained node.
+- **Deadlines** (`DEADLINES`): queued 1 h, checking 30 min, draining 30 min,
+  settling 15 min, updating 90 min, rebooting 30 min, undraining 10 min,
+  recovering 15 min. A minute tick (`host_agent_maintenance`) times out a job
+  past its deadline.
+- **A maintenance window** scoped to the host's agent metrics
+  (`host_agent/<host>`) opens when the job starts, lasts as long as the rest of
+  the job could take, and ends when it does.
+- Ingest applies all of this in a savepoint of its own: a job going wrong is
+  logged and never costs the host its samples.
+
+On the host's Agent panel: a running job leads what the host has waiting, with
+since-when; a finished one says what it did ("Done: 12 packages updated,
+rebooted"). Under Manage agent, **Update now** (with "never reboot") starts a
+job, or the panel says why one can't start.
+
+### A maintenance cadence: a group, one host at a time (#5768)
+
+`plugins/host_agent/cadence.py`, driven by the same minute tick as the jobs.
+A **plan** (`host_maintenance_plans`) names an inventory group, a weekday, a
+time in a time zone (the creator's, by default), a window and a reboot policy:
+`if_required` (the default), `always` or `never`. The page is **Hosts →
+Maintenance** (`/plugins/host_agent/maintenance`).
+
+- **When.** A plan's slot is the latest weekday-and-time at or before now, in
+  its zone. A slot fires once (`last_slot_at`) and only while its window is
+  open, so a Steward that was down over the slot starts late, not a day late.
+  A new or resumed plan starts from the next slot. **Run now** starts a run
+  without touching the schedule (the Ansible ruling on Run now).
+- **A run** (`host_maintenance_runs`) fixes its hosts at the start, by name:
+  the group's hosts through their linked Ansible targets. A host with no agent,
+  or no maintenance helper, is skipped and listed.
+- **One at a time.** Each host gets a #5767 job (`run_id` on the job). The next
+  host starts only when the last is **back in service**: its agent posting in
+  the last 3 min and, for a swarm node, the node `ready`/`active` and every
+  replicated service at its replicas, judged only from swarm reports made
+  after the job finished. A host not back within 30 min fails the run, and
+  its job is marked failed with the reason.
+- **Stops.** A failed or timed-out job fails the run. No host is started after
+  the window; the run then stops and says how many weren't reached.
+- **Alerting** is an ordinary alert rule on `host_agent/<host>/maintenance_failed`:
+  1 when the host's latest job (in 30 days) failed, timed out or didn't come
+  back into service, 0 once a later one succeeds.
+- The page leads with a running run and its hosts, then failed runs from the
+  last week, then each plan with its schedule, next slot and last five runs.
+  Each host's Agent panel shows its last maintenance job.
+
+#### Spreading swarm services back out (1.26.0, `rebalance.py`)
+
+A run drains each swarm node in turn, and the swarm never moves a task back
+by itself, so every run used to end with the services on the node drained
+first. A plan with **Spread swarm services back out** (on by default) ends a
+run that drained a node with a run-level step, once its last host is back in
+service:
+
+- **What moves.** While the fullest active node has two or more tasks than
+  the emptiest, a replicated service whose running tasks are all on the
+  fullest moves; one with several replicas there goes first. A service is
+  left where it is, and the run says why, when the plan keeps it in place
+  (**Keep in place**, service names), when it is below its replicas, when its
+  image isn't pinned to a digest, or when the emptier node doesn't hold that
+  digest. Only what kept the nodes from evening out is reported.
+- **The image check.** Agent 1.26.0 reports each service's pinned digest and
+  every `repo@sha256:…` its host holds (from `/system/df`, every 10 min; a
+  list older than 30 min isn't trusted). A service moves only to a node that
+  already holds its exact image, so a pruned image or a digest the registry
+  has since deleted can't strand it.
+- **Who moves them.** A swarm manager's helper, through a fifth step action,
+  `rebalance`, carrying the service names (each checked against Docker's name
+  alphabet, at most 48). It force-updates each in turn (`docker service update
+  --force --detach=false`, spec unchanged, no registry lookup), skipping one a
+  constraint or preference places, gives each 5 min to settle and starts no
+  move after 40 min. The swarm starts each on its emptiest node. A stolen
+  manager's agent token can therefore also restart named services in place.
+- **How it ends.** The run stays `running` until the result arrives, then
+  ends `done` with what moved, or `failed` if a move didn't settle or the
+  manager didn't answer within 60 min. A window over the `docker` figures
+  covers the restarts, and the step goes on the timeline
+  (`maintenance.rebalanced`). With no manager on 1.26.0, or nothing to move,
+  the run ends `done` and says so.
+
+### The swarm's health, alertable (1.19.0, #5788)
+
+Every manager's agent reports the whole swarm. From each report the docker plugin (`plugins/docker/swarm_health.py`) records three swarm-wide figures on resource `swarm`, source `docker`:
+
+- `swarm_services_short`: services running fewer tasks than they want. A service scaled to 0 is never short.
+- `swarm_nodes_down`: nodes not `ready`.
+- `swarm_managers_unreachable`: managers the swarm reports as not `reachable`. Agent 1.19.0 sends `ManagerStatus.Reachability`; before that, a manager whose status isn't `ready` counts.
+
+Drained nodes are left out of the last two. A node is drained on purpose (VDNT-Util01 always, any node during its maintenance), so its going away is expected, and its host's own "Agent offline" rule still covers it.
+
+The figures are recorded at most once every 30 s, whichever manager sent them, so three managers make one series and losing one doesn't silence it.
+
+A maintenance job on a swarm node opens a second window over `docker` / `swarm` beside its host's (`maintenance._swarm_window`). An alert the drain or reboot causes then waits for the job, and one still firing when the window ends is sent then.
+
+### Drive health (1.20.0, #5789; 1.20.1, #5806)
+
+Reading SMART needs root, so the root helper does it, as the one action on its fixed list that only reads: `drives`. Hourly, and at start, the agent leaves a `drives` request. The helper runs `smartctl --scan-open` to find the drives itself (the request names none), reads each with `smartctl --json -a -d <type> <dev>`, and writes a summary to `/var/lib/steward-maint/drives.json` (0644). It writes no job result, because a drive read is not a maintenance job. The agent's newest copy rides every sample, as updates do.
+
+smartctl ≥ 7.0 answers in JSON, the same on every distribution and for SATA, SAS and NVMe drives. A missing or older smartctl is reported (`smartctl not installed`, `… too old`) and its text output is never parsed. The helper reports figures only:
+
+- `passed` and `temp_c` for every drive;
+- `critical_warning`, `life_used_pct`, `media_errors` and spare % against threshold for NVMe;
+- reallocated / pending / offline-uncorrectable sectors for ATA, and life used where the drive reports it;
+- grown defects for SCSI.
+
+A drive with no SMART (a virtual disk, a bridge that hides it) reads `available: false`, and one smartctl couldn't open carries `error`. Disks behind a RAID controller share one path (`/dev/bus/0`) and differ by the slot in their type (`megaraid,3`), so the slot is part of the name: `bus0-megaraid3` (1.20.1, #5806). Steward also gives a repeated name an ordinal (`0#2`), so two drives never share a series.
+
+Steward judges the figures in `drives.py`, so the judgment can change without an agent update:
+
+- **failing**: the drive's self-assessment fails, an NVMe critical warning is raised, or spare blocks fall under threshold;
+- **worn**: at least 90% of its rated life is used;
+- **watch**: it has bad sectors or media errors but otherwise passes. Old disks carry a few for years, so a count alone isn't a failure.
+
+The figures become metrics on `<host>:drive:<device>`: `drive_failing`, `drive_life_used_pct`, `drive_bad_sectors` and `drive_temp_c`. The host gets `drives_failing` and `drive_life_used_max_pct` for covering rules; they are recorded only when at least one drive was read, so a VM's virtual disks don't claim "0 failing". The registration keeps the newest report (`drive_health`), and the host page lists it under **Drive health**, what needs attention first.
+
+**Installing smartctl on any common distribution.** `files/install-deps.sh` installs smartmontools with whichever package manager the host has: apt (without recommends, which would pull in a mail server), dnf, yum, zypper, pacman or apk. install.sh and the install / provision / update playbooks all fetch and run that one copy. A package that can't be installed is reported and never fails the install.
+
+### What happened to the host (1.14.0, #5756)
+
+A post may carry `events`, said once each, with the next post that gets through:
+
+- `started` — the agent starting, with `boot_at` (from `/proc/uptime`).
+- `stopping` — on SIGTERM, with `host_shutdown`: true, false, or null when systemd can't say.
+- `lost_contact` — `from` the first post that got no answer (or a proxy's 5xx) `to` the post carrying it.
+
+Steward turns them into the host's downtime (`host_downtime`): **reboot** (a stop, then a boot after it), **agent_stop** (the host stayed up), **unclean** (a boot after the agent was last heard that nobody announced: power loss or a crash) and **lost_contact**. An announced stop explains the silence for `reboot_grace_seconds` (Settings → Plugins → host_agent, 10 minutes): `seconds_since_seen` reads 0 until then, and the host's other rules go quiet with it, as when it is offline. Past the grace, agent offline fires. The events live in memory, so a span the agent saw is lost if it restarts mid-outage; the boot time still reads the restart as unclean.
+
+### Which temperature sensors are thermometers (#5782, #5783)
+
+A board's spare thermistor inputs (a Nuvoton's SYSTIN, AUXTIN*) are often wired to nothing, and the kernel reports them anyway, at whatever the open input reads. The host's temperature (`temp_c_max`) leaves a sensor out when:
+
+- **its value is impossible** (at or below 0 °C, or at or above 125 °C; `counts_as_hottest`), or
+- **it doesn't behave like a thermometer.** Once an hour, `sensors.refresh` judges each sensor against the host's CPU (Intel `Package id N`, AMD `Tctl`/`Tdie`/`TccdN`, else `PECI Agent N`), using the last 7 days of hourly averages. It needs at least 72 shared hours and a CPU that moved at least 10 °C. A sensor is ruled out when it **falls as the CPU warms** (Pearson r ≤ −0.5, the shape of an open input reading the chip's own reference) or **holds within 0.5 °C** while the CPU moved. A wired board or drive sensor that barely follows the CPU (r near 0) stays in.
+
+The verdicts are kept in memory and rebuilt at startup, so ingest and the host page read them without a query. The host page lists the sensors left out under "Not counted", each with its reason. A host with no CPU reading, or too little history, falls back to the value rule alone.
+
+`temp_c_max` can drive an alert rule (#5787), e.g. "Host running hot": all hosts, over 85 °C for 10 consecutive checks. So can each sensor's `temp_c`, and the per-interface and per-disk throughput. `tests/core/test_metric_catalog.py` holds every metric ingest writes to the alert catalog, except the few `NOT_ALERTABLE` names in `plugins/host_agent/__init__.py`.
+
 ### Logging
 
 To stderr only (systemd captures to journal). No file logging, no log rotation. Levels:
@@ -130,7 +416,7 @@ The agent's self-reported hostname in the payload is advisory. The real identity
 ### Failure behavior
 
 - POST fails → push sample to ring buffer (max 20), exponential backoff `30s → 60s → 120s → 300s cap`, resets on first success.
-- On next success → flush all buffered samples in a single batched POST.
+- On next success → flush the buffer oldest first, five samples a POST (`MAX_SAMPLES_PER_POST`). The new sample queues behind the backlog, so Steward always receives a host's samples in the order they were taken (1.22.0, #5995): a rule counting checks in a row then sees a stall's run of breaching checks unbroken. Before, each post carried the newest sample too, and after the 2026-10-17 NFS stall its healthy reading ended every post's run, so "Mount not answering" (10 in a row) never fired.
 - Ring buffer full → drop oldest, keep newest. Logged at DEBUG (expected during extended outages).
 - Malformed payload rejection (400) → drop the sample (don't re-buffer, it'll just be rejected again). Indicates agent/server version skew.
 - Token rejection (401) → log ERROR with remediation hint, continue retry loop at backoff cadence. Admin fixes via UI + conf file edit; no restart needed.
@@ -157,7 +443,6 @@ plugins/host_agent/
 │       └── host_agent_001_initial.py    # creates host_agent_registrations table
 ├── templates/
 │   ├── detail.html                      # per-host detail page
-│   ├── widget_table.html                # dashboard fleet table widget
 │   ├── widget_history.html              # dashboard history chart widget
 │   ├── settings_list.html               # plugin settings: list of registered hosts
 │   └── install.sh.j2                    # install script template (Jinja)
@@ -206,6 +491,8 @@ One `ScheduledTask` running every 60 seconds that flags `HostAgentRegistration`
 
 **Note:** the existing alerts system can fire on "no metric for N seconds" which is arguably a cleaner signal. The scheduled task may end up redundant once alert rules exist. Build it anyway as a minimal safety net; it's one query plus one update.
 
+**As built:** the task records `host_agent/<host>/seconds_since_seen` each minute, and "Agent offline" is an ordinary alert rule on it (`> 180`). It counts from the later of `last_seen_at` and the agent's newest post to *arrive* (kept in memory, `scheduler.arrivals`). `last_seen_at` is written in the ingest's own transaction, so a post Steward received but couldn't store (its storage stalled, #5514) would otherwise read as the agent's silence. Nothing counts from before Steward last started watching without a break (#5441).
+
 ### `METRIC_CATALOG` registration
 
 Add to `steward/alerts/routes.py`:
@@ -263,7 +550,7 @@ Content-Type: application/json
 
 ### Design points
 
-- **`samples` is always a list**, even for a single sample. This is how the ring buffer drains after a failure — agent POSTs `[current, ...buffered]` in a single request. One-path server code.
+- **`samples` is always a list**, even for a single sample. This is how the ring buffer drains after a failure — agent POSTs the oldest buffered samples, with the current one only once the backlog is through. One-path server code.
 - **`ts` is agent-reported, ISO-8601 UTC.** Server trusts it for `PluginMetric.recorded_at`. Host clocks drift but NTP is reliable enough for our purposes.
 - **If `|ts - now| > 5 minutes`**, server logs a WARN but still accepts. Don't reject, don't lose data over clock skew.
 - **`metadata` is sent on every POST**, not just on change. Server-side diff detects actual changes and only writes on change. Cost per POST is one dict — negligible. Benefit: server can cleanly detect agent restarts.
@@ -339,92 +626,17 @@ sudo sh install.sh
 
 ### Rendered script structure (Jinja template `install.sh.j2`)
 
-```sh
+`plugins/host_agent/templates/install.sh.j2` is the script; it is not copied
-#!/bin/sh
+here, because a copy drifts. It checks it runs as root with systemd and
-# Steward host agent installer
+python3, handles `--uninstall`, creates the `steward-agent` user, fetches
-# Generated for: {{ host_name }}  ({{ host_address }})
+`agent.py`, writes the config, then fetches the maintenance helper and the
-# Steward URL: {{ url }}
+three units from `/plugins/host_agent/files/` (the same files the playbooks
-set -e
+install), creates the maintenance directories, and enables
-
+`steward-maint.path` and the agent.
-STEWARD_URL="{{ url }}"
-AGENT_TOKEN="{{ token }}"
-AGENT_VERSION="{{ agent_version }}"
-
-AGENT_USER="steward-agent"
-AGENT_DIR="/usr/local/lib/steward-agent"
-CONF_FILE="/etc/steward-agent.conf"
-UNIT_FILE="/etc/systemd/system/steward-agent.service"
-
-# ── preflight ────────────────────────────────────────────────────────────────
-[ "$(id -u)" = "0" ] || { echo "must run as root (use sudo)"; exit 1; }
-command -v systemctl >/dev/null 2>&1 || { echo "systemd not found — unsupported init system"; exit 1; }
-command -v python3   >/dev/null 2>&1 || { echo "python3 not found — install python3 first"; exit 1; }
-
-# Handle --uninstall
-if [ "${1:-}" = "--uninstall" ]; then
-    systemctl disable --now steward-agent.service 2>/dev/null || true
-    rm -f "$UNIT_FILE" "$CONF_FILE"
-    rm -rf "$AGENT_DIR"
-    systemctl daemon-reload
-    # keep the system user — removing it risks orphaned files elsewhere
-    echo "uninstalled"
-    exit 0
-fi
-
-# ── create system user ───────────────────────────────────────────────────────
-if ! id "$AGENT_USER" >/dev/null 2>&1; then
-    useradd --system --no-create-home --shell /usr/sbin/nologin "$AGENT_USER"
-fi
-
-# ── drop the agent file ──────────────────────────────────────────────────────
-mkdir -p "$AGENT_DIR"
-curl -sSL "${STEWARD_URL}/plugins/host_agent/agent.py" -o "$AGENT_DIR/agent.py"
-chmod 0755 "$AGENT_DIR/agent.py"
-chown root:root "$AGENT_DIR/agent.py"
-
-# ── write config ─────────────────────────────────────────────────────────────
-cat > "$CONF_FILE" <<EOF
-url = $STEWARD_URL
-token = $AGENT_TOKEN
-interval_seconds = 30
-EOF
-chown "root:$AGENT_USER" "$CONF_FILE"
-chmod 0640 "$CONF_FILE"
-
-# ── write systemd unit ───────────────────────────────────────────────────────
-cat > "$UNIT_FILE" <<EOF
-[Unit]
-Description=Steward host agent
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-User=$AGENT_USER
-ExecStart=/usr/bin/python3 $AGENT_DIR/agent.py
-Restart=on-failure
-RestartSec=10
-NoNewPrivileges=yes
-ProtectSystem=strict
-ProtectHome=yes
-PrivateTmp=yes
-ReadOnlyPaths=/proc /sys
-
-[Install]
-WantedBy=multi-user.target
-EOF
-
-systemctl daemon-reload
-systemctl enable --now steward-agent.service
-
-echo
-echo "Steward host agent $AGENT_VERSION installed and running."
-echo "Check status:  systemctl status steward-agent"
-echo "Logs:          journalctl -u steward-agent -f"
-```
 
 ### Design points
 
+- **One copy of every file a host installs.** The units and the maintenance helper live in `plugins/host_agent/files/` and are served at `/plugins/host_agent/files/<name>` (a fixed list; anything else is 404). install.sh and the install, provision and update playbooks all fetch them, so the unit can't differ between a host installed by hand and one installed by Ansible (#5766).
 - **Agent binary served by Steward itself.** `GET /plugins/host_agent/agent.py` serves the script bundled with the plugin, so version drift between install-script and agent is impossible — re-running the installer picks up whatever agent the currently-running Steward ships.
 - **Systemd hardening is cheap and correct.** `NoNewPrivileges`, `ProtectSystem=strict`, `ProtectHome`, `PrivateTmp`, `ReadOnlyPaths=/proc /sys`. The agent only needs read access to `/proc`, `/sys`, and mount points; denying everything else narrows blast radius.
 - **Uninstall is a first-class flag**, not a separate script. Same one-liner with `--uninstall` appended.
@@ -438,7 +650,7 @@ echo "Logs:          journalctl -u steward-agent -f"
 
 ### Dashboard widgets (`steward/core/widgets.py`)
 
-- **`host_resources`** — table widget. One row per monitored host: name, CPU %, mem %, disk % (worst mount), load 1m, "last seen Xs ago" with red/yellow/green coloring. Fleet glance.
+- The fleet glance (CPU, mem, disk /, load 1m, stale mark) is the core **Hosts — Overview** widget (`hosts_overview`); the old `host_resources` table was folded into it (#5213).
 - **`host_resource_history`** — chart widget for one host. CPU / mem / disk over a selectable time range (1h, 6h, 24h, 7d). Same pattern as every other history widget — Chart.js.
 
 ### Per-host detail page

docs/plugins/index.yaml.example

@@ -1,113 +0,0 @@
-# steward-plugins / index.yaml
-#
-# This file is the catalog index for the steward plugin repository.
-# It is fetched by the app's Settings → Plugins → Plugin Catalog UI.
-#
-# Steward reads this file from:
-#   https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml
-#
-# After adding or updating a plugin entry, commit and push — the change is
-# live immediately for anyone whose app fetches the catalog (cache TTL: 5 min).
-#
-# REQUIRED fields for each plugin entry:
-#   name            - must match the plugin's directory name exactly
-#   version         - semver string
-#   download_url    - direct URL to a zip file containing the plugin
-#   checksum_sha256 - SHA-256 hex digest of the zip (leave empty to skip verification)
-#
-# OPTIONAL fields:
-#   description, author, license, min_app_version,
-#   repository_url, homepage, tags
-#
-# Generating a checksum:
-#   sha256sum traefik.zip
-#   shasum -a 256 traefik.zip   (macOS)
-#
-# Download URL conventions:
-#   Gitea release assets (recommended):
-#     https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip
-#   Upload the zip as a release attachment in Gitea; paste the URL here.
-#   Source archive tarballs work too but release assets are preferred (smaller, plugin-only).
-
-version: 1
-updated: "2026-03-22"
-
-plugins:
-
-  - name: http
-    version: "1.0.0"
-    description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry"
-    author: "Steward"
-    license: "MIT"
-    min_app_version: "0.1.0"
-    repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
-    homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/http"
-    download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/http-v1.0.0/http.zip"
-    checksum_sha256: ""
-    tags:
-      - monitoring
-      - http
-      - synthetic
-      - uptime
-
-  - name: docker
-    version: "1.0.0"
-    description: "Docker container status, resource usage, and restart tracking via Docker socket"
-    author: "Steward"
-    license: "MIT"
-    min_app_version: "0.1.0"
-    repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
-    homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker"
-    download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v1.0.0/docker.zip"
-    checksum_sha256: ""
-    tags:
-      - containers
-      - docker
-      - infrastructure
-
-  - name: traefik
-    version: "1.0.0"
-    description: "Traefik reverse proxy metrics and access log integration"
-    author: "Steward"
-    license: "MIT"
-    min_app_version: "0.1.0"
-    repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
-    homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik"
-    download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip"
-    checksum_sha256: ""   # fill in after running: sha256sum traefik.zip
-    tags:
-      - proxy
-      - metrics
-      - access-log
-
-  - name: unifi
-    version: "1.0.0"
-    description: "UniFi Network controller integration — WAN health, devices, clients, DPI"
-    author: "Steward"
-    license: "MIT"
-    min_app_version: "0.1.0"
-    repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
-    homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi"
-    download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/unifi-v1.0.0/unifi.zip"
-    checksum_sha256: ""
-    tags:
-      - network
-      - unifi
-      - ubiquiti
-
-  - name: host_agent
-    version: "1.0.0"
-    description: "Remote Linux host resource monitoring via a lightweight Python push agent"
-    author: "Steward"
-    license: "MIT"
-    min_app_version: "0.1.0"
-    repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
-    homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/host_agent"
-    download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/host_agent-v1.0.0/host_agent.zip"
-    checksum_sha256: ""
-    tags:
-      - host
-      - monitoring
-      - cpu
-      - memory
-      - storage

docs/plugins/prometheus-widgets.md

@@ -0,0 +1,210 @@
+# Prometheus widget templates
+
+The Prometheus dashboard widget draws one scrape target through a **template**: a short YAML file that says which of the target's series to show and how. Steward ships with one, **Fabled Archives**. You can write your own, paste in one someone shared, and share yours as a file.
+
+A template is data, not code. It names series and lays them out in **blocks**, and Steward draws them. It cannot add a colour, a chart or a formula. Every widget it draws has the same shape:
+
+1. **A verdict:** "All well.", or the worst thing and how many need a look ("2 need a look. Replication: FullNightly-Restic is running, 11h 53m overdue.").
+2. **Each block in turn**, its problems first, saying why, then the healthy rest. After six healthy rows a block says "+N more healthy".
+3. **The target's name and when it was scraped**, linking to its page.
+
+This page covers where templates live, a walk through a first template, each block with an example, the format reference, and the Fabled Archives template.
+
+---
+
+## Where templates live
+
+**Prometheus → Widget templates** lists every template: the built-in ones, and your own. Open one to see its YAML beside a **preview**, which draws it over a target you pick from the values Steward has already stored. The preview redraws as you type and lists each problem with its line number.
+
+| You want to | Do this |
+|---|---|
+| Start from scratch | **New template**, then write or paste YAML. |
+| Use one someone sent you | **New template**, then paste it or **upload** the `.yaml` file. Look at the preview, then **Save template**. |
+| Change a built-in one | Open it, then **Duplicate**. Built-in templates are read-only and change with Steward, so you edit the copy. |
+| Share yours | **Export** downloads the YAML as a file. |
+| Use it on a dashboard | Edit the dashboard, **Add widget → Prometheus**, and pick the target and the template. |
+
+A template saves only when it has no problems, and its `name` must differ from every other template's. A widget remembers which template it draws with, so renaming a template keeps its widgets. Deleting one leaves its widgets saying the template is gone. Your templates are in Steward's backups, and adding, changing or deleting one is recorded in the audit log.
+
+Steward never fetches a template from anywhere. Templates arrive only by being pasted or uploaded.
+
+---
+
+## Your first template
+
+Look at the target's page first (**Prometheus → the target**). It lists every series Steward stored, with its labels and newest value. A template names those series exactly as listed there.
+
+```yaml
+name: My service
+version: 1
+expects: myservice_
+blocks:
+  - values:
+      title: Health
+      items:
+        - label: Queue depth
+          metric: myservice_queue_depth
+          unit: count
+          warn_above: 100
+        - label: Disk free
+          metric: myservice_disk_free_bytes
+          of: myservice_disk_size_bytes
+          warn_below: 10%
+          crit_below: 5%
+  - times:
+      items:
+        - label: Last good run
+          metric: myservice_last_success_timestamp_seconds
+          warn_after: 26h
+          crit_after: 2d
+```
+
+- `name` is what the widget is called, and what the template list and the widget picker show.
+- `version` is the template's own version. Raise it when you change a template you have shared, so the people you sent it to can tell copies apart.
+- `expects` is the start of the metric names it reads. If the target reports none, the widget says so ("is this the template for it?") instead of a column of blanks.
+
+**Metric names as Steward stores them.** A gauge keeps its name. A counter is stored as a rate per second: `requests_total` becomes `requests_per_sec`. A histogram keeps only `_sum_per_sec` and `_count_per_sec`. See [How series are stored](prometheus.md#how-series-are-stored).
+
+---
+
+## The blocks
+
+Every block takes an optional `title`. Without one, it gets the default title shown in its heading below.
+
+### `states`: a state set, one row per thing (default title "States")
+
+A state set has one series per possible state, with the current one at 1 and the rest at 0. Archives' `fabledarchives_backup_target_state{target, state}` is one. Each thing it describes becomes a row naming its current state.
+
+```yaml
+- states:
+    title: Backup targets
+    metric: fabledarchives_backup_target_state
+    name: target                 # the label that names each row
+    healthy: [fresh]
+    quiet: [disabled, waiting]   # shown greyed, never a problem
+    warning: [late]              # anything not listed is a problem
+    columns:
+      - {label: Last backup, metric: fabledarchives_backup_target_last_backup_timestamp_seconds}
+      - {label: Stored, metric: fabledarchives_backup_target_stored_bytes}
+```
+
+- A state in `healthy` is fine, in `quiet` is shown greyed, and in `warning` is amber. **A state in none of the lists is red.** So a state the service adds later shows up as a problem rather than slipping by as healthy.
+- A row reporting no current state is a warning.
+- `columns` add the row's other series beside its state. Each column series is matched to its row by having the same labels. Failing that, it is matched by the label `name` points at.
+- **Deadlines:** `warn_past` and `crit_past` each name a `*_timestamp_seconds` metric per row. Once that time has passed, a row in a healthy state reads as a warning, or a problem for `crit_past` ("is running, 11h 53m overdue"). Quiet and failing rows keep their own reading. Use this when a service ranks a state like "running" above lateness, so a run that never ends would otherwise read as healthy.
+
+### `values`: figures with their units (default title "Figures")
+
+```yaml
+- values:
+    title: Storage
+    items:
+      - label: Disk free
+        metric: fabledarchives_data_disk_free_bytes
+        of: fabledarchives_data_disk_size_bytes   # shown as "16.2 TiB of 41.9 TiB", with a bar
+        warn_below: 10%                            # a share of `of`
+        crit_below: 5%
+        sparkline: true                            # the last day as small bars beside it
+      - label: Stored
+        metric: fabledarchives_person_stored_bytes
+        name: person                               # one row per person
+        where: {kind: files}                       # only the series with kind=files
+```
+
+- `warn_above`, `crit_above`, `warn_below` and `crit_below` take a number, or a share like `10%` when the item has `of`. The red thresholds are checked before the amber ones.
+- Without `name`, an item must match exactly one series. If more match, the row says how many and asks you to narrow it with `where` or `name`.
+- A series the target doesn't report reads "—", "not reported", as a warning.
+- The sparkline's bars are scaled to that series' own peak over the last 24 hours.
+
+### `times`: moments and how long ago (default title "Times")
+
+```yaml
+- times:
+    items:
+      - label: Last good run
+        metric: myservice_last_success_timestamp_seconds
+        warn_after: 26h
+        crit_after: 2d
+```
+
+The metric is a `*_timestamp_seconds`. The row reads "3h ago" (exact time on hover) and turns amber or red once that long has passed. A timestamp of 0 reads "never", which is a warning when the item has a threshold.
+
+### `table`: one metric, two labels (default title "Table")
+
+```yaml
+- table:
+    title: Stored per person
+    metric: fabledarchives_person_stored_bytes
+    rows: person
+    columns: kind
+```
+
+Rows and columns are the values of the two labels, in alphabetical order, under one header row. A missing cell reads "—". Tables show figures only; they don't judge them.
+
+### `info`: labels worth reading (default title "About")
+
+```yaml
+- info:
+    title: Running
+    metric: fabledarchives_build_info
+    labels: [version, channel, commit]   # which labels, in this order; all of them when left out
+```
+
+This block is for a `*_info` or `*_build_info` series, where the labels are the point and the value is always 1. It reads "version 2026.10.12.1717 · channel dev · commit 3605d0a".
+
+---
+
+## Reference
+
+**Top level**
+
+| Key | Needed | What it is |
+|---|---|---|
+| `name` | yes | At most 80 characters; unique among your templates and the built-in ones. |
+| `version` | yes | A whole number from 1. |
+| `blocks` | yes | One to 20 blocks. |
+| `description` | | A line shown in the template list. |
+| `expects` | | The prefix of the metric names it reads, like `fabledarchives_`. |
+| `format` | | The template format; `1`, the only one so far. |
+
+**Keys most blocks and items take**
+
+| Key | What it is |
+|---|---|
+| `metric` | A series name as Steward stores it. |
+| `where` | Labels a series must have, like `{kind: files, state: ok}`. |
+| `name` | On `states`, `values` and `times`: the label that names each row (one row per value). |
+| `unit` | On a `values` item, a `states` column or a `table`: overrides the unit read from the name: `bytes`, `seconds` (a span), `percent`, `per_sec`, `count` (thousands separated: 100,462), `timestamp`. |
+
+Without `unit`, the name decides how a value reads: `_bytes` as bytes, `_seconds` as a span, `_timestamp_seconds` as a moment, `_per_sec` as a rate, `_pct` as a percentage. Anything else is a plain number.
+
+**Spans** (`warn_after`, `crit_after`) are a number of seconds or a number with `s`, `m`, `h` or `d`: `90`, `15m`, `26h`, `2d`.
+
+**Limits:** a template is at most 64 KB, with at most 20 blocks and 30 items in a block.
+
+**What the checker refuses,** each with its line and what to change:
+- an unknown key;
+- a key given twice;
+- anchors and aliases (`&name`, `*name`): write the value out;
+- a share threshold (`10%`) without `of`;
+- a state in two of `healthy`, `quiet` and `warning`;
+- `crit_after` earlier than `warn_after`;
+- a word YAML reads as true or false (`on`, `off`, `yes`, `no`) used as a state or label value. Quote it: `'off'`.
+
+---
+
+## The Fabled Archives template, walked through
+
+The built-in template is a worked example of most of the above. **Duplicate** it to read or change it. It was written against the series Archives actually reports. On 14 Oct 2026 that turned up a nightly replication job that had read "running" for two days without a good copy; the deadlines below exist to catch exactly that.
+
+| Block | What it shows | Why it is written this way |
+|---|---|---|
+| Backup targets (`states`) | Each target Archives receives backups from: fresh, late or missing, with its last backup and what it has stored. | `disabled` and `waiting` are quiet: a target you turned off, or one that hasn't sent its first backup, isn't a problem. `late` is a warning; `missing` (not listed) is a problem. |
+| Replication (`states`) | Each replication job, its last good copy and its next run. | Archives ranks `running` above lateness, so `warn_past` and `crit_past` read the job's own late and missing times. A run that never finishes turns amber, then red. |
+| Storage and integrity (`values`) | Disk free of its size (amber under 10%, red under 5%, last day as bars), files checked of files, malware held, failing notification channels. | Held malware and a failing channel each need you to do something, so any count above 0 is a warning. |
+| Stored per person (`table`) | Each person's files, versions and trash. | Figures to read, not to judge. |
+| Running (`info`) | The version, channel and commit. | |
+
+Left out on purpose (the template's header says so too):
+- `fabledarchives_section_up`: a section Archives couldn't read stops reporting its figures, and the rows that read them then say "not reported".
+- The background queue, HTTP requests and audit counters: these suit charts and alert rules better than a glance.

docs/plugins/prometheus.md

@@ -0,0 +1,113 @@
+# Prometheus Plugin
+
+The Prometheus plugin scrapes any service's Prometheus `/metrics` endpoint and stores what you choose to keep in Steward's metrics. Once stored, a series can be charted, read through the MCP `query_metrics` tool, and watched by an alert rule like any other metric.
+
+It is on by default. Targets are added, tested and edited on its page, **Prometheus** in the sidebar (`/plugins/prometheus/`).
+
+---
+
+## Adding a target
+
+| Field | What it is |
+|---|---|
+| Name | How the target's series are named in Steward (rule pickers, charts). No `/`, `:` or comma. |
+| Metrics address | The full URL, e.g. `http://archives:8081/metrics`. |
+| Bearer token | Sent as `Authorization: Bearer …`. Stored encrypted and never shown again; leave it blank to keep the saved one. |
+| Scrape every | 10 s to an hour. Default 60 s. |
+| Check its HTTPS certificate | Off only for a self-signed certificate you trust. |
+| Keep / Leave out | Name patterns, one a line, `*` for any run of characters. Empty Keep keeps everything; Leave out wins over Keep. |
+| Series cap | The most series stored from one scrape (default 1000). Past it, the rest are counted and shown on the target's page, not stored. |
+
+**Test** scrapes the address once with what is typed, filters included, and says how many series would be stored and under which names. Nothing is saved.
+
+The endpoint hands over everything; Steward chooses what to keep. Start narrow (`yourservice_*`) and widen it if you find you want more: every stored series is a row every scrape.
+
+---
+
+## How series are stored
+
+| On the endpoint | In Steward |
+|---|---|
+| A gauge or an untyped sample | Stored as it is, under its own name. |
+| A counter (`requests_total`) | Stored as a rate per second, `requests_per_sec`, from the second scrape on. A value lower than the last (the service restarted) is skipped rather than read as a negative rate. |
+| A histogram or summary | Its `_sum` and `_count` are stored as rates (`x_sum_per_sec`, `x_count_per_sec`). Buckets and quantiles are not stored. |
+| NaN or ±Inf | Not stored. |
+
+A series without labels is stored under the target's name. A labelled one is stored under `<target>:<label>=<value>,…`, in the order the endpoint gives the labels, for example `Archives:target=Steward,state=late`.
+
+Each target also records four metrics about itself, under its own name:
+
+| Metric | Meaning |
+|---|---|
+| `is_up` | 1 when the last scrape got metrics, 0 when it didn't. |
+| `scrape_ms` | How long the scrape took. |
+| `series_kept` | Series stored from the last scrape. |
+| `series_capped` | Series the last scrape left out because of the cap. |
+
+A value's unit is read from Prometheus naming: `_bytes` as bytes, `_seconds` as a span, `_timestamp_seconds` as a moment ("3h ago"), `_per_sec` as a rate.
+
+**A refused token stops the scraping.** When an endpoint answers 401 or 403, Steward stops asking and says so on the target's page and in the overview, because asking again won't change the answer and some services, Home Assistant among them, ban an address after a few refusals. Saving the target, usually with a new token, or pausing and resuming it, starts it again. Any other failure is retried at the next scrape.
+
+A **state set** is one series per possible state, the current one 1 and the rest 0, such as `fabledarchives_backup_target_state{target, state}`. The target page shows it as one row per target (or job), naming the current state; the other states are listed on hover. A metric is read as a state set when its name ends in `_state` and every series has a `state` label reading 0 or 1. The OpenMetrics form, where the label is named after the metric (`foo{foo="a"} 1`), counts too. Each state is still stored as its own series, so an alert rule can watch one state.
+
+**Only what the endpoint reports now is current.** When a scrape succeeds, any series it no longer has loses its newest value. That happens when the endpoint stopped reporting it, for example an old `build_info` after an upgrade, or when a filter or the cap now leaves it out. Its history stays on the chart until retention removes it. A failed scrape leaves the newest values as they were.
+
+Renaming a target moves its stored history to the new name. Deleting one removes its newest values; its history ages out with retention (Settings → Retention).
+
+---
+
+## The dashboard widget
+
+Edit a dashboard, open **Add widget**, and pick **Prometheus** under Integrations. Choose a **target** and a **template**; the widget is named after the template unless you give it a title.
+
+A template says which of the target's series to show and how: backup states, figures with their units, times, a table, the version running. The widget leads with one line naming the worst problem, then each section with its problems first and the healthy rest after. Its **Details** link opens the target's page.
+
+Steward ships with one template, **Fabled Archives**. A target whose series don't match its template says so in the widget rather than showing blanks.
+
+## Widget templates
+
+**Prometheus → Widget templates** lists every template the widget can draw with: the ones built in, and your own. A template of yours that no longer passes the checker (a newer Steward may read the format differently) is listed first with what to change, and the widgets drawn with it say so too.
+
+Open one to see its YAML beside a **preview**: the template drawn over a target you choose, exactly as the widget draws it, from the values Steward has stored. Nothing is fetched from the target to draw it.
+
+- **New template**: write one, or paste in one someone shared. The preview redraws as you type, and lists each problem with its line until there are none.
+- **Upload a file**: on the same page, read a `.yaml` file into the editor to look over before saving.
+- **Duplicate**: copy any template, built-in ones included, under a new name, and change the copy. Built-in templates are read-only: they ship with Steward and change with it.
+- **Export**: download a template's YAML, to keep or to share.
+
+How to write one, every block, and the format reference: [Prometheus widget templates](prometheus-widgets.md).
+
+Saving needs a template that passes the checker and a name no other template has. A widget remembers which template it draws with, so renaming a template keeps its widgets; deleting one leaves its widgets saying the template is gone. Templates are kept in Steward's backups, and adding, changing or deleting one is in the audit log.
+
+## Alerting on a target
+
+In **Alerts → New rule**, choose **Prometheus targets**, then the series (the resource) and the metric. Two rules worth having for any target:
+
+- **It stopped answering:** resource `<target>`, metric `is_up`, `< 1`.
+- **Something was cut off:** resource `<target>`, metric `series_capped`, `> 0`.
+
+---
+
+## Recipe: Fabled Archives
+
+Archives serves `/metrics` on its backup listener (port 8081), which is reachable only on your LAN and the swarm's overlay network, behind a scrape token.
+
+1. In Archives, **Settings → Metrics → Make a token**, named e.g. "Steward". Copy it; it is shown once.
+2. In Steward, **Prometheus → Add target**:
+   - Name `Archives`
+   - Metrics address `http://<archives host>:8081/metrics` (on a shared overlay network, the service name, e.g. `http://archives_app:8081/metrics`)
+   - Bearer token: the one you copied
+   - Keep: `fabledarchives_*`
+3. **Test**, then **Add target**.
+
+Rules worth adding:
+
+| Watch | Resource | Metric | Condition |
+|---|---|---|---|
+| A backup target is late | `Archives:target=<name>,target_id=<id>,state=late` | `fabledarchives_backup_target_state` | `>= 1` |
+| A backup target is missing | `Archives:target=<name>,target_id=<id>,state=missing` | `fabledarchives_backup_target_state` | `>= 1` |
+| A replication job is failing | `Archives:job=<name>,job_id=<id>,state=failing` | `fabledarchives_replication_job_state` | `>= 1` |
+| Archives' data disk is filling | `Archives` | `fabledarchives_data_disk_free_bytes` | `<` your floor |
+| Archives stopped answering | `Archives` | `is_up` | `< 1` |
+
+The target's page lists every stored series with its exact resource name, so copy names from there rather than typing them.

docs/plugins/syslog.md

@@ -0,0 +1,88 @@
+# Syslog plugin
+
+Receives syslog from NASes, routers and other devices over UDP and TCP
+(RFC 3164 and RFC 5424). The plugin keeps the lines of the sources you add and
+puts them beside Steward's other readings.
+
+## What it does
+
+- **Sources.** A device that starts sending is *offered* on the Syslog page.
+  Its lines are counted there but not stored until an admin adds it. Ignored
+  senders are still counted. Each added source keeps its lines in a ring:
+  the age limit (`retention_days`, 14 by default) and the size cap
+  (`max_bytes_per_source`) are set in Settings → Plugins → Syslog.
+- **Reading.** Each source has its own page, with lines filtered by severity,
+  app, text and time. A host's page and an SNMP device's page show the newest
+  lines of the sources mapped to them. A source maps to a host by its binding,
+  or, when it has none, by sharing the host's or device's address.
+- **Patterns.** Syslog → Patterns counts each source's lines matching some
+  phrases (case-insensitive) or a severity over a window. The count is written
+  every minute as the metric `lines_matching_<key>` under source module
+  `syslog`, zeros included, so an ordinary alert rule fires on the first match
+  and clears once the window has passed. Ready-made patterns cover disk I/O
+  errors, disk resets, NFS server trouble, filesystem errors, degraded arrays
+  and "errors and worse". "Alert on it" opens the rule form aimed at the count.
+- **Beside the readings.** `get_anomaly` includes the warnings and worse logged
+  by the anomaly's host or SNMP device. Each anomaly row has "Syslog around
+  it". Host metrics charts and SNMP device charts mark each error with a
+  dashed red line.
+- **MCP.** `list_syslog_sources`, `search_syslog`, `list_syslog_patterns` and `add_syslog_preset`.
+
+## Deploying
+
+Steward runs as an unprivileged user, so it listens on **5514** (UDP and TCP),
+not 514. Publish that port from the container:
+
+```yaml
+# docker compose
+ports:
+  - "5514:5514/udp"
+  - "5514:5514/tcp"
+```
+
+Under **Docker Swarm**, either publishing mode works.
+
+**Ingress** (Swarm's default) lets Steward move between nodes. Devices send to
+any node's address, or to an address shared over the nodes (a keepalived VIP):
+
+```yaml
+ports:
+  - { target: 5514, published: 5514, protocol: udp }
+  - { target: 5514, published: 5514, protocol: tcp }
+```
+
+Through ingress every line arrives from Docker's own address on the ingress
+network, so the address no longer tells the devices apart. For such an address
+Steward keys the source by the hostname each line carries instead (#6096): each
+device is its own source. A device must put its hostname in its lines (NASes
+and routers do); lines with none at Docker's address all fold into one source,
+and the page says so. Docker's address changes when Steward's task moves to
+another node; a named source stays the same source, added or not, and takes the
+new address.
+
+Such a source's own address, which places its lines on its SNMP device's and
+host's pages (chart marks, the lines around an anomaly), is found by name: the
+SNMP device whose reported name (sysName, asked with every poll) or given name
+matches, else the host of that name. Where nothing matches, set **Device
+address** on the source's edit page; a value set there always wins.
+
+**Host mode** keeps each device's own address, but only the node running the
+task answers, so devices must be pointed at that node:
+
+```yaml
+ports:
+  - { target: 5514, published: 5514, protocol: udp, mode: host }
+  - { target: 5514, published: 5514, protocol: tcp, mode: host }
+```
+
+To let devices use 514 instead, publish `514:5514` instead of `5514:5514`.
+
+## Pointing a device at it
+
+The Syslog page's "Point a device here" section gives the address and port,
+and where the setting is on common devices (Synology, ASUSTOR, QNAP, TrueNAS,
+OpenWrt, UniFi, rsyslog). The address is the plugin's `send_to` setting
+(Settings → Plugins → Syslog): Steward's web address is often a name only its
+proxy answers, so it isn't guessed. Both formats are read, BSD (RFC 3164) and
+IETF (RFC 5424); choose IETF where it is offered, as its lines carry their time
+zone. ASUSTOR's ADM offers no choice and sends BSD.

docs/plugins/traefik.md

@@ -12,7 +12,20 @@ On each scrape:
 3. Writes computed metrics to the `traefik_metrics` history table
 4. Calls `record_metric()` for each metric, making them available to alert rules
 
-Request rates and error rates are computed as deltas between the current and previous scrape, divided by elapsed time. Latency percentiles (p50, p95, p99) are approximated via linear interpolation over histogram buckets — these are estimates, not exact percentiles.
+Request rates and error rates are computed as deltas between the current and previous scrape, divided by elapsed time. Latency percentiles (p50, p95, p99) are approximated via linear interpolation over histogram buckets — these are estimates, not exact percentiles. They cover the responses since the previous scrape (each bucket's rise, summed over every status code and method), not every response since Traefik started. A scrape in which a router answered nothing records no latency, shown as "—"; one past the last bucket (5 s by default) reads as that bucket's edge.
+
+When the metrics URL can reach more than one Traefik (a swarm service's replicas behind the routing mesh or its VIP, a Kubernetes Service's pods, a keepalived pair after a failover, or the same Traefik after a restart), each process keeps its own counters. Steward tells them apart by `process_start_time_seconds` and compares each scrape only with the same process's previous one. The first scrape from a process it hasn't seen in the last hour sets that process's baseline and counts nothing. When `metrics_url`'s host name resolves to several addresses, Steward scrapes each one and adds the replicas' requests, bytes and latency buckets together, so the figures are the whole proxy's. Each replica still counts only against its own last scrape; one that doesn't answer is left out with a warning in the log. Certificates are listed once however many replicas serve them; Traefik's own figures add open connections and memory and show the newest reload.
+
+Through a name with one address (a swarm service's VIP, a Kubernetes ClusterIP, a load balancer) you get whichever replica answered, which can be one that serves no traffic at all. Point `metrics_url` at a name that lists every replica instead:
+
+| Where Traefik runs | `metrics_url` |
+|---|---|
+| One container | its address, e.g. `http://traefik:8080/metrics` |
+| Docker Swarm service (Steward on a network it shares with Traefik) | `http://tasks.<service>:8080/metrics`, e.g. `http://tasks.traefik_router:8080/metrics` |
+| Kubernetes | a headless Service's name |
+| A pair behind a keepalived VIP | a name that resolves to both nodes |
+
+`api_url` can stay on one address: every replica serves the same configuration.
 
 ---
 
@@ -20,7 +33,7 @@ Request rates and error rates are computed as deltas between the current and pre
 
 | Key | Default | Description |
 |---|---|---|
-| `metrics_url` | `http://localhost:8080/metrics` | Traefik Prometheus endpoint |
+| `metrics_url` | `http://localhost:8080/metrics` | Traefik Prometheus endpoint. A host name with several addresses is scraped address by address and added up (see above) |
 | `scrape_interval_seconds` | `60` | How often to scrape |
 
 Enable the plugin by setting `enabled: true` in the app settings (Settings UI or directly in `app_settings` DB table under key `plugin.traefik`).
@@ -79,7 +92,22 @@ The widget auto-refreshes via HTMX polling every `poll_interval` seconds. Fragme
 
 ### Detail Page
 
-The full Traefik page at `/plugins/traefik/` shows all routers with sparkline history charts (last 20 data points) for request rate, p95 latency, and 5xx error rate.
+The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time range picked at the top:
+
+1. **The verdict.** It names the routers that need a look, says why, and says when the problem started ("api, 3.0% of responses are 5xx, started 2h ago"). When a router has been a problem for the whole range, it says so instead. Otherwise it reads "All N routers healthy".
+2. **The request chart.** Requests per second across all routers and the share answered 5xx, the same chart as the dashboard widget.
+3. **A card for each router that needs a look:** why, since when, the 4xx and 5xx status codes it answered in the last 15 minutes, most first ("404 ×1,812, 502 ×40"), and its request rate, bandwidth, p95/p99 latency and 4xx/5xx shares, each with its history. The code is what separates scanners probing for pages (404), a login wall (401/403) and an app that isn't answering (502/503/504). Steward records codes from the version that added them on; until then a card says they aren't recorded yet.
+   Each card then says what its problem usually means and what to do (`plugins/traefik/meaning.py`, #5712):
+   - **Server errors (5xx)** are almost always real. Mostly 500: the app is failing requests. Mostly 502/503/504: Traefik couldn't reach the app, or it timed out.
+   - **Client errors (4xx)** are usually traffic rather than a fault. Mostly 404: scanners and bots. 401/403: a login wall. 429: rate limiting. Mostly 499 is the exception: clients giving up on a slow app.
+   - **Slow responses** compare p95 with p50. When p95 is 4× p50 or more, a few requests are slow (one endpoint or a heavy page). When p50 is close behind, the whole app is slow.
+   - **Next:** a fault links to the swarm service behind the router (its tasks and logs) when Steward can match the name ("codex@swarm" → `codex_app`, "radarr@swarm" → `arr_radarr`), otherwise to Docker → Swarm. Traffic links to Settings → Thresholds.
+4. **Traefik's own figures:** when its config last reloaded, open connections, memory.
+5. **The healthy routers,** folded into one table, then the routers Traefik no longer reports, then TLS certificates by soonest expiry, then Traffic Origin when the access log is on. A certificate Traefik has renewed is listed as old, not by its expiry, though a Traefik that hasn't restarted since keeps reporting it: one whose every name a later-expiring certificate covers.
+
+A router is graded on its last 15 minutes against Settings → Thresholds. When a problem started is read from the router's history over the range.
+
+Once a router has 5 days of history, its 4xx share and p95 latency are judged against its own normal for that hour (see [Normals and anomalies](../architecture.md)) instead of the fixed cutoffs. One over the cutoff but within its normal is healthy, and the healthy table says it is usual for that router (an error-pages service that answers 4xx all day, a login wall's steady 401s). One out of its normal is a warning that says what is usual, and a metric anomaly to review; the Traefik page lists the routers' anomalies of the last 30 days. How far out counts is set on Settings → Thresholds → Normals (2 points of 4xx share, 50 ms of p95 by default). A 4xx or 5xx share over fewer requests in the 15 minutes than Settings → Thresholds allows (30 by default) isn't judged at all: two 500s of sixteen requests would otherwise read 12% and critical. A 5xx share past its cutoff is failing only once the router answered at least 3 server errors in those 15 minutes (Settings → Thresholds, "Fewest server errors to fail"): one 502 among 70 requests is 1.4%, but it is one error. The 5xx share is a failure and keeps its fixed cutoff whatever the router's history. Latency is learned only from #5755's per-scrape figures, so its normal arrives 5 days after that release.
 
 ---
 

docs/plugins/writing-a-plugin.md

@@ -28,7 +28,7 @@ license: "MIT"          # any SPDX identifier, e.g. MIT, Apache-2.0, GPL-3.0
 # Optional: prevents loading on older app versions
 min_app_version: "0.1.0"
 
-# Optional: shown in the catalog UI
+# Optional: where the plugin's source lives
 repository_url: "https://github.com/yourname/yourrepo"
 homepage: "https://github.com/yourname/yourrepo/tree/main/myplugin"
 tags:
@@ -323,6 +323,21 @@ Role hierarchy: `admin > operator > viewer`. Requiring `viewer` grants access to
 
 ---
 
+## Audit Log Entries
+
+A change made through a plugin's pages is audited with `steward.core.audit.log_audit`, naming the thing it changed by `entity_type` and `entity_id`. So the audit page shows that thing's current name, linked to its page, export `get_audit_entities()` from `__init__.py`:
+
+```python
+# plugins/myplugin/__init__.py
+def get_audit_entities() -> dict:
+    from .models import Target
+    return {"myplugin_target": (Target, "/plugins/myplugin/targets/{}")}
+```
+
+The model needs `id` and `name` columns. An entity deleted since keeps the name the entry recorded in its detail (`detail={"name": ...}`). Add the action's wording to `ACTIONS` in `steward/audit/service.py` so the row reads as a sentence.
+
+---
+
 ## Agent Tools (MCP)
 
 Steward serves an MCP endpoint at `/mcp` that AI clients reach with a personal API token created on the AI access page. A plugin adds tools to it by exporting an optional `register_mcp_tools(register)` from `__init__.py`:
@@ -376,85 +391,33 @@ async def mcp_overview(db) -> list[dict]:
 - `severity` is `"critical"` (something is down or about to be) or `"warning"` (degraded, or heading that way). Report problems only; a healthy plugin returns `[]`.
 - It is called only while the plugin is enabled, with its own session. If it raises, the overview still returns, and names the plugin under `unavailable`.
 
+### What happened on a host
+
+`get_anomaly` shows, beside a metric anomaly, what else happened on its host around each occurrence. A plugin that records events on a host adds them by exporting an optional `mcp_events(db, host_id, host_name, since, until)` that returns events built with `steward.mcp.tools._events.event`:
+
+```python
+# plugins/myplugin/__init__.py
+async def mcp_events(db, host_id, host_name, since, until) -> list[dict]:
+    from steward.mcp.tools._events import event
+    return [event(r.at, "backup_job", f"Backup {r.name}: {r.state}", until=r.finished_at)
+            for r in await _jobs_on(db, host_id, since, until)]
+```
+
+- Return what happened between `since` and `until`, oldest first: something that lasted gives `until`. Bound your query (a few hundred rows); Steward keeps the 40 nearest the occurrence's start and tells the caller how many it left out (`around_left_out`).
+- It is called only while the plugin is enabled, with its own session. If it raises, the anomaly is still returned without the plugin's events. The host agent (maintenance jobs, reboots) and Docker (container starts, deaths, OOM kills) are the examples to read.
+
+A resource with no host (a Traefik router, say) has no host to read events by. A plugin that knows what happened to one of its own resources exports `mcp_resource_events(db, source, resource, since, until)` instead, same return shape, and returns `[]` for a source that isn't its own. Traefik's is the example: it gives a router's swarm service's container events, on whichever hosts they ran.
+
+Core adds, beside every anomaly, the stretches Steward itself wasn't recording (kind `"steward"`: a restart, a deploy, a stall), read from its own once-a-minute health sample; a restart is shown however quick, from the `restart_gap_secs` its new process records. A plugin doesn't need to report those.
+
 ---
 
-## Publishing to the Catalog
+## Shipping a Plugin
 
-The official plugin catalog is hosted at `https://git.fabledsword.com/bvandeusen/Steward-plugins`. Anyone can submit a plugin by opening a pull request — first-party and third-party plugins are treated identically by the catalog system.
+Steward never downloads plugins. There is no catalog and no install button: the app runs with no outbound network.
 
-### Repo layout
+- **First-party plugins** live in this repository under `plugins/` and ship inside the image, so they version with core.
-
+- **A third-party plugin** is a folder you put in the external plugin root, `/data/plugins` by default (`STEWARD_PLUGIN_DIR` moves it). Restart Steward and the plugin appears in Settings → Plugins, where you enable it. A bundled plugin of the same name wins over an external one.
-```
-steward-plugins/
-├── index.yaml          ← catalog index — the only file Steward fetches
-├── myplugin/
-│   ├── plugin.yaml
-│   ├── __init__.py
-│   └── ...
-└── .github/
-    └── workflows/
-        └── publish.yml ← packages each plugin dir into a zip on release
-```
-
-### index.yaml entry
-
-Each plugin needs a corresponding entry in `index.yaml`. See `docs/plugins/index.yaml.example` for the full schema. The minimum required fields are:
-
-```yaml
-- name: myplugin
-  version: "1.0.0"
-  description: "What this plugin does"
-  author: "Your Name"
-  license: "MIT"
-  min_app_version: "0.1.0"
-  repository_url: "https://github.com/yourname/yourrepo"
-  homepage: "https://github.com/yourname/yourrepo/tree/main/myplugin"
-  download_url: "https://github.com/yourname/yourrepo/releases/download/myplugin-v1.0.0/myplugin.zip"
-  checksum_sha256: ""   # fill in after zipping — see below
-  tags:
-    - monitoring
-```
-
-### Creating a release zip
-
-The zip must contain the plugin files at the top level OR inside a single directory named after the plugin. Both layouts work:
-
-```
-# Layout A — flat (preferred)
-myplugin.zip
-├── plugin.yaml
-├── __init__.py
-└── ...
-
-# Layout B — single top-level directory (also accepted, GitHub archive default)
-myplugin.zip
-└── myplugin/
-    ├── plugin.yaml
-    └── ...
-```
-
-Generate the zip and its checksum:
-
-```bash
-cd steward-plugins
-zip -r myplugin.zip myplugin/
-sha256sum myplugin.zip   # paste this into index.yaml checksum_sha256
-```
-
-Upload `myplugin.zip` as a GitHub release asset, then update `index.yaml` with the release download URL and checksum.
-
-### How install works
-
-When a user clicks **Install** in Settings → Plugins:
-
-1. The app downloads the zip from `download_url`
-2. Verifies the SHA-256 checksum (if provided)
-3. Extracts the plugin into its `PLUGIN_DIR`
-4. Runs any pending Alembic migrations for the plugin
-5. Attempts a **hot-reload** — registers the blueprint and scheduled tasks without restarting
-6. If the plugin was previously loaded (blueprint already mounted), a **restart** is required to pick up the new code
-
-Hot-reload works reliably for brand-new plugin installs. Updates to already-active plugins require a restart, which can be triggered from the same settings page.
 
 ---
 
@@ -471,4 +434,3 @@ Hot-reload works reliably for brand-new plugin installs. Updates to already-acti
 - [ ] MCP tools (if any) registered via `register_mcp_tools(register)` with a role and `read_only`
 - [ ] Templates namespaced under `templates/myplugin/`
 - [ ] Plugin enabled in app settings
-- [ ] `index.yaml` entry added with `download_url` and `checksum_sha256`

plugins/docker/__init__.py

@@ -11,7 +11,7 @@ _app: "Quart | None" = None
 def setup(app: "Quart") -> None:
     global _app
     _app = app
-    from .models import DockerContainer, DockerMetric  # noqa: F401 — register with Base
+    from .models import DockerContainer  # noqa: F401 — register with Base
 
     # Publish the per-host persist hook so the host agent can hand off the
     # `docker` array from each sample without importing our models (opportunistic
@@ -21,27 +21,55 @@ def setup(app: "Quart") -> None:
     from steward.core.capabilities import register_capability
     from steward.models.users import UserRole
     from .ingest import persist_host_docker
-    from .retention import run_docker_retention
+    from .retention import prune_docker_logs, run_docker_retention
     register_capability(
         "docker.persist_host_samples", persist_host_docker,
         label="Persist host Docker samples",
         description="Store per-host container state + metrics pushed by the host agent.",
         required_role=UserRole.viewer,
     )
-    # Roll up + prune Docker time-series, driven by the core cleanup task without
+    # Prune lifecycle events, driven by the core cleanup task without it
-    # it importing our models. Same trusted server-side data-plane role as above.
+    # importing our models. Same trusted server-side data-plane role as above.
+    # Container CPU and memory live in the one metric store and its retention.
     register_capability(
         "docker.run_retention", run_docker_retention,
         label="Run Docker retention",
-        description="Roll up old docker_metrics to hourly + prune stale metrics/events.",
+        description="Prune Docker lifecycle events past their window.",
         required_role=UserRole.viewer,
     )
+    register_capability(
+        "docker.prune_logs", prune_docker_logs,
+        label="Prune Docker logs",
+        description="Keep each container's logs within the age window and byte cap.",
+        required_role=UserRole.viewer,
+    )
+    # Each container's CPU (100% is one core) and memory share learn their
+    # normal (#5742), each with its floor from Settings → Thresholds. Memory
+    # is its processes' own, file cache left out (#6030).
+    from steward.core.norms import floor_setting, learn_normal
+    learn_normal("docker", "cpu_pct", floor_setting(app, "container_cpu_pct"))
+    learn_normal("docker", "mem_used_pct", floor_setting(app, "container_mem_pct"))
+    # A host's agent carries its containers' figures: they go silent with it
+    # (#5721). A swarm service's aren't tied to one host.
+    from steward.core.alerts import agent_fed
+    from .service import SERVICE_RESOURCE
+    agent_fed("docker", lambda resource: None if resource.startswith(SERVICE_RESOURCE)
+              else resource.split("/", 1)[0])
+    # An alert on a swarm figure names the services or nodes it counts (#6400).
+    from steward.core.alerts import names_behind
+    from .swarm_health import NAMERS
+    for metric, names in NAMERS.items():
+        names_behind("docker", metric, names)
+    # Deploys and crash-loops go on the timeline.
+    from .timeline import register as register_timeline
+    register_timeline()
 
 
 def get_scheduled_tasks() -> list:
-    # Collection is now agent-driven (pushed to the host_agent ingest); the
+    # Collection is agent-driven (pushed to the host_agent ingest). The one
-    # central socket scrape was removed. No periodic task to register.
+    # task here moves task-named history to its service.
-    return []
+    from .rekey import make_rekey_task
+    return [make_rekey_task(_app)]
 
 
 def get_blueprint():
@@ -59,13 +87,36 @@ async def mcp_overview(db) -> list[dict]:
     return await findings(db)
 
 
+async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
+    from .mcp_tools import mcp_events as events
+    return await events(db, host_id, host_name, since, until)
+
+
+async def mcp_resource_events(db, source: str, resource: str, since, until) -> list[dict]:
+    from .service import SERVICE_RESOURCE
+    if source != "docker" or not resource.startswith(SERVICE_RESOURCE):
+        return []
+    from .mcp_tools import service_events
+    return await service_events(db, resource[len(SERVICE_RESOURCE):], since, until)
+
+
 def get_nav() -> list[dict]:
     # Surfaces in the sidebar's Infrastructure group (fleet-wide container view).
     return [{"label": "Docker", "href": "/plugins/docker/"}]
 
 
 def get_alert_metrics() -> dict[str, list[str]]:
-    """What ingest.py records per container (resource "<host>/<container>"):
+    """What ingest.py records per container (resource "swarm/<service>" for a
-    restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 /
+    swarm task, else "<host>/<container>"): restart_count is cumulative (alert
-    0.0 from the container's HEALTHCHECK, only for containers that define one."""
+    on crash-looping); is_healthy is 1.0 / 0.0 from the container's HEALTHCHECK,
-    return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy"]}
+    only for containers that define one. net_rx_bps / net_tx_bps are the bytes
+    a second it receives and sends, under "<host>/<service or container>"
+    (#6115): "a container flooding the uplink" is net_tx_bps over a level.
+    Per host (resource "<host>"), what Docker keeps on its disk (#5304).
+    Swarm-wide (resource "swarm"), from any manager's report: services short
+    of their replicas, nodes down and managers unreachable, drained nodes
+    aside (#5788, swarm_health.py)."""
+    from .disk_view import DISK_METRICS
+    from .swarm_health import SWARM_METRICS
+    return {"docker": ["cpu_pct", "mem_used_pct", "restart_count", "is_healthy", "net_rx_bps", "net_tx_bps",
+                       *DISK_METRICS, *SWARM_METRICS]}

plugins/docker/disk_view.py

@@ -0,0 +1,82 @@
+# plugins/docker/disk_view.py
+"""What Docker keeps on each host's disk, over time and by image repository
+(#5304–#5306). Model-free, so it is unit-tested without a database.
+
+The agent reports `docker system df` as current state; ingest.py records its
+totals as metrics too (DISK_METRICS, resource = the host's name), so each
+host's growth has a history and an alert rule can watch it.
+"""
+from __future__ import annotations
+
+from datetime import datetime
+
+# metric name → the df summary field it records; "disk_total_bytes" is their sum.
+DISK_PARTS = {
+    "disk_images_bytes": "images_size",
+    "disk_containers_bytes": "containers_size",
+    "disk_volumes_bytes": "volumes_size",
+    "disk_build_cache_bytes": "build_cache_size",
+}
+DISK_METRICS = [*DISK_PARTS, "disk_total_bytes", "disk_reclaimable_bytes"]
+
+
+def disk_metrics(disk: dict) -> dict[str, float]:
+    """The metric samples one df report gives: each part, their total, and
+    what unused images hold."""
+    out = {m: float(int(disk.get(f) or 0)) for m, f in DISK_PARTS.items()}
+    out["disk_total_bytes"] = sum(out.values())
+    out["disk_reclaimable_bytes"] = float(int(disk.get("images_reclaimable") or 0))
+    return out
+
+
+def growth(points: list[tuple[datetime, float]]) -> dict | None:
+    """How much a total changed over the points given (oldest first): the
+    change, the first point's time and value. None with fewer than two."""
+    if len(points) < 2:
+        return None
+    (first_at, first), (_, last) = points[0], points[-1]
+    return {"change": last - first, "since": first_at, "from": first, "to": last,
+            "pct": (last - first) / first * 100 if first else None}
+
+
+def image_repo(repo_tag: str | None) -> tuple[str | None, str | None]:
+    """("ghcr.io/x/app", "1.2") for "ghcr.io/x/app:1.2"; a registry's port is
+    part of the repository ("reg:5000/app" has no tag); a digest reference
+    keeps its digest as the tag; (None, None) for an untagged image."""
+    if not repo_tag or repo_tag == "<none>" or repo_tag.startswith("<none>"):
+        return None, None
+    if "@" in repo_tag:
+        repo, _, digest = repo_tag.partition("@")
+        return repo, digest[:19]
+    head, sep, tag = repo_tag.rpartition(":")
+    if not sep or "/" in tag:
+        return repo_tag, None
+    return head, tag
+
+
+def group_images(images: list[dict]) -> list[dict]:
+    """Images folded under their repository, the most space first.
+
+    Each image: {repo_tag, image_id, size, shared_size, containers}. A group's
+    `own` counts each version's size less the layers it shares with other
+    images, so a repository is not charged twice for one base layer;
+    `reclaimable` is that, for the versions no container uses. Untagged images
+    form one group of their own (repo None)."""
+    groups: dict[str | None, dict] = {}
+    for im in images:
+        repo, tag = image_repo(im.get("repo_tag"))
+        g = groups.setdefault(repo, {"repo": repo, "versions": [], "own": 0, "size": 0,
+                                     "reclaimable": 0, "containers": 0})
+        own = max(int(im.get("size") or 0) - int(im.get("shared_size") or 0), 0)
+        used = int(im.get("containers") or 0)
+        g["versions"].append({"tag": tag, "image_id": im.get("image_id"), "size": int(im.get("size") or 0),
+                              "shared": int(im.get("shared_size") or 0), "own": own, "containers": used})
+        g["own"] += own
+        g["size"] = max(g["size"], int(im.get("size") or 0))
+        g["containers"] += used
+        if not used:
+            g["reclaimable"] += own
+    for g in groups.values():
+        g["versions"].sort(key=lambda v: (v["containers"] == 0, -v["size"]))
+        g["unused"] = sum(1 for v in g["versions"] if not v["containers"])
+    return sorted(groups.values(), key=lambda g: (g["repo"] is None, -g["own"], g["repo"] or ""))

plugins/docker/ingest.py

@@ -30,6 +30,41 @@ def _parse_started_at(value) -> datetime | None:
         return None
 
 
+def net_rates(before: dict, snapshots: list) -> list:
+    """(recorded_at, name, rx_bps, tx_bps) for each running container in each
+    snapshot, from its cumulative network counters against the last ones seen
+    for it: `before` is {name: (at, container_id, rx, tx)} from the stored rows,
+    and each snapshot moves it on. A container new since then, recreated under
+    the same name, or whose counters went back (restarted) has no rate until
+    its next snapshot (#6115)."""
+    last = dict(before)
+    out = []
+    for at, containers in snapshots:
+        for c in containers:
+            name, rx, tx = c.get("name"), c.get("net_rx_bytes"), c.get("net_tx_bytes")
+            if not name or c.get("status") != "running" or rx is None or tx is None:
+                continue
+            cid = c.get("container_id") or ""
+            prev = last.get(name)
+            last[name] = (at, cid, rx, tx)
+            if prev is None or prev[1] != cid or prev[2] is None or prev[3] is None:
+                continue
+            dt = (at - prev[0]).total_seconds()
+            if dt <= 0 or rx < prev[2] or tx < prev[3]:
+                continue
+            out.append((at, name, (rx - prev[2]) / dt, (tx - prev[3]) / dt))
+    return out
+
+
+def _crash_loop_deaths() -> int:
+    from quart import current_app
+    from .service import CRASH_LOOP_DEATHS, crash_loop_deaths
+    try:
+        return crash_loop_deaths(current_app.config)
+    except RuntimeError:                    # outside the app (a unit test)
+        return CRASH_LOOP_DEATHS
+
+
 def _derive_events(old_state: dict, new_containers: list) -> list:
     """Diff a fresh snapshot against stored per-container state → lifecycle events.
 
@@ -147,12 +182,36 @@ async def _persist_logs(session, host, log_batches) -> None:
     if not await get_setting(session, "docker.logs.enabled"):
         return
     exclude = set(await get_setting(session, "docker.logs.exclude") or ())
-    for row in _log_rows(log_batches, host.id):
+    rows = [r for r in _log_rows(log_batches, host.id) if r["container_name"] not in exclude]
-        if row["container_name"] in exclude:
+    stored = await _stored_lines(session, host.id, rows)
+    for row in rows:
+        if (row["container_name"], row["ts"], row["stream"], row["line"]) in stored:
             continue
         session.add(DockerLog(**row))
 
 
+async def _stored_lines(session, host_id: str, rows: list) -> set:
+    """The lines of `rows` this host already has, as (container, ts, stream,
+    line). A restarted agent has lost its cursors and re-sends each
+    container's last lines (#5809); a push of new lines overlaps nothing, so
+    the one query, bounded per container to the batch's own span, finds none."""
+    from sqlalchemy import and_, or_, select
+    from .models import DockerLog
+
+    spans: dict[str, list] = {}
+    for r in rows:
+        span = spans.setdefault(r["container_name"], [r["ts"], r["ts"]])
+        span[0], span[1] = min(span[0], r["ts"]), max(span[1], r["ts"])
+    if not spans:
+        return set()
+    found = await session.execute(
+        select(DockerLog.container_name, DockerLog.ts, DockerLog.stream, DockerLog.line)
+        .where(DockerLog.host_id == host_id, or_(*(
+            and_(DockerLog.container_name == name, DockerLog.ts.between(lo, hi))
+            for name, (lo, hi) in spans.items()))))
+    return {tuple(r) for r in found}
+
+
 async def _persist_swarm(session, host, swarm: dict) -> None:
     """Upsert this manager's swarm topology; drop rows no longer reported.
 
@@ -180,6 +239,8 @@ async def _persist_swarm(session, host, swarm: dict) -> None:
         row.desired = int(s.get("desired") or 0)
         row.running = int(s.get("running") or 0)
         row.image = s.get("image")
+        digest = s.get("image_digest")
+        row.image_digest = digest[:80] if isinstance(digest, str) else None
         row.placement_json = json.dumps(s.get("placement") or [])
         row.tasks_json = json.dumps(s.get("tasks") or [])
         row.updated_at = now
@@ -239,7 +300,18 @@ async def _persist_disk(session, host, disk: dict) -> None:
     summary.images_active = int(disk.get("images_active") or 0)
     summary.containers_count = int(disk.get("containers_count") or 0)
     summary.volumes_count = int(disk.get("volumes_count") or 0)
+    digests = disk.get("digests")
+    summary.digests_json = json.dumps(
+        [d[:512] for d in digests if isinstance(d, str)][:400] if isinstance(digests, list) else [])
     summary.updated_at = now
+    # The totals as metrics too (#5304): a history of each host's growth, and
+    # something an alert rule can watch. Resource: the host's name.
+    from steward.core.alerts import MetricBatch
+    from .disk_view import disk_metrics
+    batch = MetricBatch()
+    for metric, value in disk_metrics(disk).items():
+        batch.add(source_module="docker", resource_name=host.name, metric_name=metric, value=value)
+    await batch.record(session)
 
     images = disk.get("images") or []
     seen: set[str] = set()
@@ -284,6 +356,20 @@ async def _persist_status(session, host, status: dict) -> None:
     row.reported_at = status["at"]
 
 
+async def _record_swarm_health(session, swarm: dict) -> None:
+    """The swarm-wide figures alert rules watch (#5788), at most once every
+    RECORD_EVERY whichever manager reported."""
+    from datetime import timezone
+    from steward.core.alerts import MetricBatch
+    from . import swarm_health
+    if not swarm_health.due(datetime.now(timezone.utc)):
+        return
+    batch = MetricBatch()
+    for metric, value in swarm_health.figures(swarm).items():
+        batch.add(source_module="docker", resource_name=swarm_health.RESOURCE, metric_name=metric, value=value)
+    await batch.record(session)
+
+
 async def _expire_swarm(session, host) -> None:
     """Drop the swarm rows a host reported while it was a manager."""
     from .models import DockerSwarmNode, DockerSwarmService
@@ -298,9 +384,11 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
 
     `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
     one entry per ingested sample carrying docker data (usually one; more when
-    the agent flushes a backlog). Every snapshot contributes time-series
+    the agent flushes a backlog). Every snapshot's running containers record
-    DockerMetric rows; the newest snapshot drives current container state, the
+    cpu_pct, mem_used_pct, mem_used_bytes and mem_cache_bytes into the one
-    alert pipeline, and lifecycle-event derivation. `swarm` is the newest
+    metric store, at the snapshot's time, and net_rx_bps / net_tx_bps from
+    their counters (net_rates); the newest snapshot drives current container state and
+    lifecycle-event derivation. `swarm` is the newest
     sample's swarm object (or None off managers) — persisted when present.
     `disk` is the newest sample's /system/df summary (or None on Docker-less
     hosts) — persisted when present. `logs` is a list of (recorded_at, records)
@@ -315,7 +403,8 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
     flushed before its mark, so a slow ingest names the part (#5181).
     """
     from steward.core.alerts import MetricBatch
-    from .models import DockerContainer, DockerEvent, DockerMetric
+    from .models import DockerContainer, DockerEvent
+    from .service import container_resource, traffic_resource
 
     async def mark(name: str) -> None:
         if phases is not None:
@@ -329,6 +418,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
         await mark("status")
     if swarm is not None:
         await _persist_swarm(session, host, swarm)
+        await _record_swarm_health(session, swarm)
         await mark("swarm")
     if disk is not None:
         await _persist_disk(session, host, disk)
@@ -342,26 +432,40 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
     ordered = sorted(snapshots, key=lambda s: s[0])
     latest_at, latest_containers = ordered[-1]
 
-    # Time-series points for every snapshot (running containers with a CPU read).
+    # Time series for every snapshot (running containers with a CPU read), in
-    for recorded_at, containers in ordered:
+    # the one metric store: resource "swarm/<service>" for a swarm task, so its
-        for c in containers:
+    # history survives the task being replaced or moved, else
-            if c.get("status") == "running" and c.get("cpu_pct") is not None:
+    # "<host>/<container>", so containers of the same name on different hosts
-                session.add(DockerMetric(
+    # don't collide. Recorded with the alert batch below, at each snapshot's
-                    host_id=host.id,
+    # own time.
-                    container_name=c["name"],
-                    scraped_at=recorded_at,
-                    cpu_pct=c["cpu_pct"],
-                    mem_pct=c.get("mem_pct") or 0.0,
-                    mem_usage_bytes=c.get("mem_usage_bytes") or 0,
-                ))
-    await mark("metrics")
-
     # Snapshot of stored per-container state BEFORE the upsert overwrites it —
-    # used to diff lifecycle events. Skip event derivation on the host's first
+    # the network counters each rate counts from, and the lifecycle-event diff.
-    # snapshot (empty state) so we don't emit a start per pre-existing container.
+    # Skip event derivation on the host's first snapshot (empty state) so we
+    # don't emit a start per pre-existing container.
     old_rows = (await session.execute(
         select(DockerContainer).where(DockerContainer.host_id == host.id)
     )).scalars().all()
+
+    batch = MetricBatch()
+    for recorded_at, containers in ordered:
+        for c in containers:
+            if c.get("status") != "running" or c.get("cpu_pct") is None or not c.get("name"):
+                continue
+            resource = container_resource(host.name, c["name"])
+            for metric in ("cpu_pct", "mem_used_pct", "mem_used_bytes", "mem_cache_bytes"):
+                if c.get(metric) is not None:
+                    batch.add(source_module="docker", resource_name=resource,
+                              metric_name=metric, value=c[metric], recorded_at=recorded_at)
+    # Bytes a second each container receives and sends, so a host's traffic
+    # can be put down to the container behind it (#6115).
+    before = {r.name: (r.scraped_at, r.container_id, r.net_rx_bytes, r.net_tx_bytes) for r in old_rows}
+    for recorded_at, name, rx_bps, tx_bps in net_rates(before, ordered):
+        resource = traffic_resource(host.name, name)
+        batch.add(source_module="docker", resource_name=resource,
+                  metric_name="net_rx_bps", value=round(rx_bps, 1), recorded_at=recorded_at)
+        batch.add(source_module="docker", resource_name=resource,
+                  metric_name="net_tx_bps", value=round(tx_bps, 1), recorded_at=recorded_at)
+    await mark("metrics")
     old_state = {
         r.name: {"status": r.status, "health": r.health,
                  "oom_killed": r.oom_killed, "exit_code": r.exit_code,
@@ -373,16 +477,22 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
         # the sample that noticed it was taken.
         finished = {c.get("name"): _parse_started_at(c.get("finished_at"))
                     for c in latest_containers}
+        derived = []
         for name, event, detail in _derive_events(old_state, latest_containers):
             at = finished.get(name) if event in ("die", "stop") else None
             session.add(DockerEvent(
                 host_id=host.id, container_name=name,
                 event=event, detail=detail, at=at or latest_at,
             ))
+            derived.append((name, event, detail, at or latest_at))
+        # Deploys and crash-loops go on the timeline, judged against the
+        # containers as they were before this snapshot.
+        from . import timeline
+        await timeline.from_snapshot(session, host, old_rows, latest_containers, derived, latest_at,
+                                     _crash_loop_deaths())
     await mark("events")
 
-    # Current state + alerts from the newest snapshot only.
+    # Current state, and the restart and health alerts, from the newest snapshot only.
-    batch = MetricBatch()
     for c in latest_containers:
         name = c.get("name")
         if not name:
@@ -395,9 +505,10 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
         existing.image = c.get("image", "") or ""
         existing.status = c.get("status", "unknown") or "unknown"
         existing.cpu_pct = c.get("cpu_pct")
-        existing.mem_usage_bytes = c.get("mem_usage_bytes")
+        existing.mem_used_bytes = c.get("mem_used_bytes")
         existing.mem_limit_bytes = c.get("mem_limit_bytes")
-        existing.mem_pct = c.get("mem_pct")
+        existing.mem_used_pct = c.get("mem_used_pct")
+        existing.mem_cache_bytes = c.get("mem_cache_bytes")
         existing.ports_json = json.dumps(c.get("ports") or [])
         existing.started_at = _parse_started_at(c.get("started_at"))
         existing.scraped_at = latest_at
@@ -416,19 +527,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
         existing.blk_read_bytes = c.get("blk_read_bytes")
         existing.blk_write_bytes = c.get("blk_write_bytes")
 
-        # Alert pipeline — resource is host-scoped so containers of the same name
+        resource = container_resource(host.name, name)
-        # on different hosts don't collide in the metric/alert namespace.
-        resource = f"{host.name}/{name}"
-        if c.get("status") == "running" and c.get("cpu_pct") is not None:
-            batch.add(
-                source_module="docker",
-                resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
-            )
-            if c.get("mem_pct") is not None:
-                batch.add(
-                    source_module="docker",
-                    resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
-                )
         # Restart count is alertable regardless of state (crash-looping matters
         # most while the container is down/restarting).
         batch.add(

plugins/docker/mcp_tools.py

@@ -12,12 +12,13 @@ from datetime import datetime, timedelta
 
 from sqlalchemy import select
 
+from steward.mcp.tools.tickets import open_tickets_for
 from steward.ansible.runner import trigger_run
 from steward.ansible.sources import BUILTIN_SOURCE_NAME
 from steward.mcp._context import current_caller, mcp_app
 from steward.mcp.tools._audit import audit_mcp
 from steward.mcp.tools._shape import (
-    age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, summarize,
+    age_s, bounded, clamp_limit, cut_middle, db_session, iso, lookback, now_utc, pick, rnd, summarize,
 )
 from steward.mcp.tools.overview import finding
 from steward.models.ansible_inventory import AnsibleTarget
@@ -27,23 +28,14 @@ from steward.models.users import UserRole
 from . import service as docker_svc
 from .service import PROBLEM_SAYS, PROBLEM_SEVERITY, PROBLEMS, container_problems, problem_since
 from .dedup import dedup_by_container_id
-from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
+from .models import DockerContainer, DockerDiskUsage, DockerEvent, DockerSwarmNode, DockerSwarmService
 from .swarm_view import build_swarm_services, replaced_tasks, task_time
 
 CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
-LOG_LINE_CHARS = 2000
 
 
 # ── pure helpers ─────────────────────────────────────────────────────────────
 
-def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
-    """Keep both ends of an over-long line and say how much of the middle went."""
-    if text is None or len(text) <= limit:
-        return text
-    keep = limit // 2
-    return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
-
-
 def container_row(c, host_names: dict, problems: list[str], now: datetime,
                   replaced: bool = False, since: datetime | None = None) -> dict:
     running = (c.status or "").lower() == "running"
@@ -62,8 +54,8 @@ def container_row(c, host_names: dict, problems: list[str], now: datetime,
         "oom_killed": bool(c.oom_killed),
         "restart_count": c.restart_count,
         "cpu_pct": rnd(c.cpu_pct),
-        "mem_pct": rnd(c.mem_pct),
+        "mem_used_pct": rnd(c.mem_used_pct),
-        "mem_usage_bytes": c.mem_usage_bytes,
+        "mem_used_bytes": c.mem_used_bytes,
         "mem_limit_bytes": c.mem_limit_bytes,
         "started_at": iso(c.started_at),
         "uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
@@ -107,10 +99,17 @@ def task_error_text(err: dict | None) -> str:
     return f"{why}{code}{where}"
 
 
+# Services that started failing within this long of each other read as one
+# finding: they are usually one node's event (#5722).
+TOGETHER = timedelta(minutes=5)
+
+
 def service_findings(services: list[dict]) -> list[dict]:
-    """Swarm services that are failing: crash-looping, under-replicated, or that
+    """Swarm services failing now: crash-looping, or short of replicas, each
-    lost a task to an error in the last 24h. Each names Swarm's last task error."""
+    naming Swarm's last task error. One back to all its replicas is history,
-    out = []
+    kept on the Docker page, not a finding (#5722). Several short of replicas
+    since within TOGETHER of each other are one finding naming them all."""
+    out, short = [], []
     for s in services:
         replicas = f"{s['running']} of {s['desired']} replicas running"
         last = task_error_text(s.get("last_error"))
@@ -124,20 +123,40 @@ def service_findings(services: list[dict]) -> list[dict]:
                 f"last hour, {replicas}{tail}", since=since, ids=ids,
                 next="list_swarm_services()"))
         elif s["state"] != "healthy":
+            short.append((s, replicas, tail))
+    for group in _together(short):
+        if len(group) == 1:
+            s, replicas, tail = group[0]
             out.append(finding(
                 "critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
                 s["name"], f"swarm service {s['name']} has {replicas}{tail}",
-                since=since, ids=ids, next="list_swarm_services()"))
+                since=s.get("trouble_since"), ids={"service_name": s["name"]},
-        elif s.get("failures_24h"):
-            n = s["failures_24h"]
-            out.append(finding(
-                "warning", "swarm_task_failed", s["name"],
-                f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
-                f"24h and is now {replicas}{tail}", since=since, ids=ids,
                 next="list_swarm_services()"))
+            continue
+        names = [s["name"] for s, _r, _t in group]
+        down = any(s["state"] == "down" for s, _r, _t in group)
+        out.append(finding(
+            "critical" if down else "warning", "swarm_services_short", f"{len(group)} swarm services",
+            f"{len(group)} swarm services started failing together and are short of replicas: "
+            + ", ".join(f"{s['name']} ({r})" for s, r, _t in group),
+            since=min(s["trouble_since"] for s, _r, _t in group),
+            ids={"service_names": names}, next="list_swarm_services()"))
     return out
 
 
+def _together(short: list[tuple]) -> list[list[tuple]]:
+    """Short services grouped by when they started failing: each group's
+    starts lie within TOGETHER of the one before. One without a start stands alone."""
+    timed = sorted((x for x in short if x[0].get("trouble_since")), key=lambda x: x[0]["trouble_since"])
+    groups: list[list[tuple]] = []
+    for x in timed:
+        if groups and x[0]["trouble_since"] - groups[-1][-1][0]["trouble_since"] <= TOGETHER:
+            groups[-1].append(x)
+        else:
+            groups.append([x])
+    return groups + [[x] for x in short if not x[0].get("trouble_since")]
+
+
 # ── loading ──────────────────────────────────────────────────────────────────
 
 async def _host_names(db) -> dict[str, str]:
@@ -173,12 +192,63 @@ async def _swarm(db, now: datetime):
     return built["services"], nodes, host_names
 
 
+def unique_nodes(nodes: list) -> list:
+    """Each swarm node once: every manager reports them all, so keep the
+    newest report of each."""
+    seen, out = set(), []
+    for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
+        if n.node_id not in seen:
+            seen.add(n.node_id)
+            out.append(n)
+    return out
+
+
+async def _maintaining(db) -> dict[str, str]:
+    """{host name, lower case: job state} for hosts with a maintenance job
+    running (#5835): a node drained or rebooting for one is expected. The
+    host agent is always loaded; None-safe all the same."""
+    from steward.core.plugin_manager import loaded_plugin_submodule
+    models = loaded_plugin_submodule("host_agent", "models")
+    maint = loaded_plugin_submodule("host_agent", "maintenance")
+    if models is None or maint is None:
+        return {}
+    job = models.HostMaintenanceJob
+    rows = await db.execute(select(Host.name, job.state).join(Host, Host.id == job.host_id)
+                            .where(job.state.in_(maint.ACTIVE)))
+    return {name.lower(): state for name, state in rows}
+
+
+NODE_AVAILABILITY_SAYS = {"drain": "drained", "pause": "paused"}
+
+
+def node_findings(nodes: list, maintaining: dict[str, str]) -> list[dict]:
+    """Swarm nodes down now (#5835): status not ready. A node whose host has
+    a maintenance job running is rebooting on purpose and isn't named.
+
+    Drained alone is not a finding: the operator keeps a manager drained on
+    purpose (#5777), and a drain a failed maintenance job left behind is that
+    job's finding."""
+    out = []
+    for n in unique_nodes(nodes):
+        name = n.hostname or n.node_id
+        if n.status == "ready" or name.lower() in maintaining:
+            continue
+        drained = NODE_AVAILABILITY_SAYS.get(n.availability)
+        out.append(finding(
+            "critical", "swarm_node_down", name,
+            f"swarm {n.role} {name} is {n.status}" + (f" and {drained}" if drained else ""),
+            ids={"node_id": n.node_id}, next="list_swarm_services()"))
+    return out
+
+
 async def mcp_overview(db) -> list[dict]:
-    """get_overview findings: failing containers and under-replicated swarm services."""
+    """get_overview findings: failing containers, under-replicated swarm
+    services, and swarm nodes that are down."""
     now = now_utc()
     rows = await _rows(db, now)
-    services, _, _ = await _swarm(db, now)
+    services, nodes, _ = await _swarm(db, now)
-    return container_findings(rows) + service_findings(services)
+    return (container_findings(rows) + service_findings(services)
+            + node_findings(nodes, await _maintaining(db)))
 
 
 # ── tools ────────────────────────────────────────────────────────────────────
@@ -290,6 +360,10 @@ async def _pick_container(db, container: str, host: str, *, removed: bool = Fals
             return RemovedContainer(*gone[0])
     names = sorted(c.name for c in partial)[:10] + [f"{r[1]} (removed)" for r in gone[:10]]
     hint = f" Containers whose names contain it: {', '.join(names)}." if names else ""
+    services = sorted({c.service_name for c in partial if c.service_name})
+    if services:
+        hint += (f" Swarm services among them: {', '.join(services[:10])}; search_container_logs "
+                 "takes a service's name and searches every task it has run.")
     raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} "
                      f"{'' if names else _REMOVED_HINT + ' '}list_containers lists them.")
 
@@ -306,9 +380,10 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
       a name also works (swarm task names are long).
     host: the host's name or id. Needed only when several hosts run a
       container with this name.
-    range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
+    range: the window for `cpu_pct_history` and `mem_used_pct_history` (min,
-      max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
+      avg, max, p95, last and a ≤60-point series) — "1h", "6h", "24h"
-      "7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
+      (default), "7d" or "30d". `cpu_pct` and `mem_used_pct` are the latest
+      reported values.
     events: how many lifecycle events (start, restart, stop, die, oom, health_change)
       to return, newest first (default 20, max 100).
 
@@ -316,7 +391,14 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
     still resolves by its exact name, or by a part of it that matches one
     removed container, while Steward retains its logs or events: the reply
     has `removed: true`, `last_seen_at`, its history and its events, and no
-    live state. A live container has `removed: false`.
+    live state. A live container has `removed: false`. `open_tickets` are the
+    open tickets about it (list_tickets).
+
+    Memory (`mem_used_bytes`, `mem_used_pct`, of its limit or of the host's
+    RAM when it has none) is what its processes hold: anon + shmem.
+    `mem_cache_bytes` is the file cache charged to it, which the kernel
+    gives back when the host needs the memory, so a large figure there is
+    not memory the container needs.
     """
     since, key = lookback(range)
     events = clamp_limit(events, 20, 100)
@@ -330,9 +412,11 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
             db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
         host_names = await _host_names(db)
         cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
-        mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
+        mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_used_pct", since)
         evs = await docker_svc.container_events(db, c.host_id, c.name, events)
         sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)])
+        tickets = await open_tickets_for(
+            db, subject=docker_svc.container_resource(host_names.get(c.host_id, c.host_id), c.name))
     hour_ago = now - timedelta(hours=1)
     recent = sum(1 for e in failures if e.at >= hour_ago)
     try:
@@ -348,10 +432,12 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
         "ports": ports,
         "net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
         "blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
+        "mem_cache_bytes": c.mem_cache_bytes,
         "range": key,
         "cpu_pct_history": summarize(cpu),
-        "mem_pct_history": summarize(mem),
+        "mem_used_pct_history": summarize(mem),
         "events": _event_rows(evs, now),
+        "open_tickets": tickets,
         "next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
     }
 
@@ -366,7 +452,7 @@ async def _removed_container(db, c: RemovedContainer, since, key: str, events: i
     """get_container for a container Docker has removed: what Steward kept."""
     host_names = await _host_names(db)
     cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
-    mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
+    mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_used_pct", since)
     evs = await docker_svc.container_events(db, c.host_id, c.name, events)
     return {
         "container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id),
@@ -374,7 +460,7 @@ async def _removed_container(db, c: RemovedContainer, since, key: str, events: i
         "last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now),
         "range": key,
         "cpu_pct_history": summarize(cpu),
-        "mem_pct_history": summarize(mem),
+        "mem_used_pct_history": summarize(mem),
         "events": _event_rows(evs, now),
         "next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")',
     }
@@ -407,21 +493,25 @@ async def list_swarm_services(limit: int = 100) -> dict:
       Swarm keeps few old tasks, so a long outage reads as a lower bound.
 
     Services with a problem come first. `nodes` lists every swarm node once,
-    with its role (manager/worker), availability and status.
+    with its role (manager/worker, `leader`), availability (active, drain,
+    pause) and status (ready, down, unknown); `host_id` is the Steward host
+    of that name (get_host, get_maintenance_job), and `maintenance` the state
+    of a maintenance job running on it, which drains and reboots it on
+    purpose (null when none).
     limit: default 100, max 500.
     """
     limit = clamp_limit(limit, 100, 500)
     now = now_utc()
     async with db_session() as db:
         services, nodes, host_names = await _swarm(db, now)
-    seen, node_rows = set(), []
+        maintaining = await _maintaining(db)
-    for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
+    host_ids = {name.lower(): hid for hid, name in host_names.items()}
-        if n.node_id in seen:
+    node_rows = [{"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
-            continue
-        seen.add(n.node_id)
-        node_rows.append({"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
                   "leader": n.leader, "availability": n.availability, "status": n.status,
-                          "reported_by": host_names.get(n.host_id)})
+                  "host_id": host_ids.get((n.hostname or "").lower()),
+                  "maintenance": maintaining.get((n.hostname or "").lower()),
+                  "reported_by": host_names.get(n.host_id)}
+                 for n in unique_nodes(nodes)]
     node_rows.sort(key=lambda n: n["hostname"].lower())
     order = {"down": 0, "degraded": 1, "healthy": 2}
 
@@ -450,7 +540,7 @@ async def list_swarm_services(limit: int = 100) -> dict:
                 {"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
                 {"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
                  "status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
-                 "cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
+                 "cpu_pct": rnd(r["cpu_pct"]), "mem_used_pct": rnd(r["mem_pct"])}
                 for r in s["replicas"]
             ],
         }
@@ -464,10 +554,14 @@ async def search_container_logs(container: str = "", host: str = "", text: str =
                                 context: int = 0, limit: int = 50) -> dict:
     """Search the container log lines Steward has retained, newest first.
 
-    container: one container's name (as list_containers shows); empty searches
+    container: one container's name (as list_containers shows), or a Swarm
-      every container (narrow with `host`). A container Docker has removed
+      service's name (as list_swarm_services shows) to search every task
-      still works while its lines are retained: the reply then has
+      container the service has run, live and removed, so a search reaches
-      `container_removed: true` and a `note` saying when it was last seen.
+      back across deploys; each match then has the `slot` and `task_id` that
+      wrote it. Empty searches every container (narrow with `host`). A
+      container Docker has removed still works while its lines are retained:
+      the reply then has `container_removed: true` and a `note` saying when it
+      was last seen.
     host: a host's name or id; empty for all hosts.
     text: case-insensitive substring to match; empty matches every line.
     range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are
@@ -496,16 +590,19 @@ async def search_container_logs(container: str = "", host: str = "", text: str =
         except ValueError as exc:
             raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc
     async with db_session() as db:
-        host_id = name = removed = None
+        host_id = name = removed = service = tasks = None
+        if host:
+            hosts = list((await db.execute(select(Host))).scalars())
+            host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
         if container:
+            service, tasks = await _as_service(db, container.strip(), host_id)
+            if service is None:
                 c = await _pick_container(db, container, host, removed=True)
                 host_id, name = c.host_id, c.name
                 removed = c if isinstance(c, RemovedContainer) else None
-        elif host:
+        rows = await docker_svc.search_logs(db, since=since, host_id=host_id, name=name,
-            hosts = list((await db.execute(select(Host))).scalars())
+                                            containers=tasks, text=text, stream=stream,
-            host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
+                                            before=before_dt, limit=limit + 1)
-        rows = await docker_svc.search_logs(db, since=since, host_id=host_id, name=name, text=text,
-                                            stream=stream, before=before_dt, limit=limit + 1)
         more = len(rows) > limit
         rows = rows[:limit]
         host_names = await _host_names(db)
@@ -513,12 +610,17 @@ async def search_container_logs(container: str = "", host: str = "", text: str =
         for r in rows:
             m = {"at": iso(r.ts), "container_name": r.container_name, "host_id": r.host_id,
                  "host_name": host_names.get(r.host_id), "stream": r.stream, "line": cut_middle(r.line)}
+            if service:
+                m["slot"], m["task_id"] = docker_svc.task_of(service, r.container_name)
             if context:
                 pre, post = await docker_svc.log_context(db, r.host_id, r.container_name, r.ts, context)
                 m["before"] = [cut_middle(x.line) for x in pre]
                 m["after"] = [cut_middle(x.line) for x in post]
             matches.append(m)
     out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more}
+    if service:
+        out["service"] = service
+        out["tasks_searched"] = len(tasks)
     if removed is not None:
         out["container_removed"] = True
         out["note"] = _removed_note(removed)
@@ -532,6 +634,24 @@ async def search_container_logs(container: str = "", host: str = "", text: str =
     return out
 
 
+async def _as_service(db, want: str, host_id: str | None):
+    """(service, its task containers) when `want` names a Swarm service rather
+    than a container (#5468): every task container Steward holds anything for,
+    live and removed, kept to `host_id` when given (none there reads as no
+    lines, not as a missing name). (None, None) when it is not a service, or
+    a container carries that exact name, which keeps precedence."""
+    if not want:
+        return None, None
+    exact = (await db.execute(select(DockerContainer.name).where(
+        DockerContainer.name == want).limit(1))).first()
+    if exact is not None:
+        return None, None
+    tasks = await docker_svc.service_containers(db, want)
+    if not tasks:
+        return None, None
+    return want, [(h, n) for h, n in tasks if host_id is None or h == host_id]
+
+
 async def get_docker_disk_usage(host: str = "") -> dict:
     """Docker disk usage per host: images, containers, volumes and build cache, in bytes.
 
@@ -635,6 +755,38 @@ async def prune_docker(host: str, what: str, check: bool = True) -> dict:
             "next": f'wait_for_run(run_id="{run.id}")'}
 
 
+# Deploys, restarts, deaths and OOM kills are on the timeline (timeline.py),
+# which get_anomaly reads; these are the container events it doesn't hold.
+_UNTIMELINED = ("start", "stop", "health_change")
+
+
+async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
+    """Container events on a host in a window (get_anomaly) that aren't on
+    the timeline: starts, stops and health changes."""
+    from steward.mcp.tools._events import event
+    rows = (await db.execute(select(DockerEvent).where(
+        DockerEvent.host_id == host_id, DockerEvent.event.in_(_UNTIMELINED),
+        DockerEvent.at >= since, DockerEvent.at <= until)
+        .order_by(DockerEvent.at).limit(500))).scalars()
+    return [event(e.at, "container", f"{e.container_name} {e.event}" + (f" ({e.detail})" if e.detail else ""),
+                  subject=f"{host_name}/{e.container_name}") for e in rows]
+
+
+async def service_events(db, service: str, since, until) -> list[dict]:
+    """What happened to a swarm service's containers on every host in a window
+    (get_anomaly): starts with their image, deaths, OOM kills, health changes.
+    A service has no one host to read events by."""
+    from steward.mcp.tools._events import event
+    from steward.models.hosts import Host
+    rows = (await db.execute(
+        select(DockerEvent, Host.name).join(Host, Host.id == DockerEvent.host_id)
+        .where(DockerEvent.container_name.startswith(f"{service}.", autoescape=True),
+               DockerEvent.at >= since, DockerEvent.at <= until)
+        .order_by(DockerEvent.at).limit(500))).all()
+    return [event(e.at, "container", f"{e.container_name} {e.event}" + (f" ({e.detail})" if e.detail else ""),
+                  subject=f"{host}/{e.container_name}") for e, host in rows]
+
+
 def register_tools(register) -> None:
     for fn in (list_containers, get_container, list_swarm_services,
                search_container_logs, get_docker_disk_usage):

plugins/docker/migrations/env.py

@@ -15,7 +15,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
 
 from steward.models.base import Base
 import steward.models  # noqa: F401
-from plugins.docker.models import DockerContainer, DockerMetric  # noqa: F401
+from plugins.docker.models import DockerContainer  # noqa: F401
 
 config = context.config
 if config.config_file_name is not None:

plugins/docker/migrations/versions/docker_012_drop_metrics.py

@@ -0,0 +1,56 @@
+"""Container CPU and memory move to the one metric store
+
+Drops docker_metrics and docker_metrics_hourly (#5752). Container cpu_pct,
+mem_pct and mem_usage_bytes are recorded into plugin_metrics as source
+"docker", resource "<host>/<container>", and roll up with every other metric.
+CPU and memory % were already recorded there for alerting, so their history
+stays; memory in bytes starts fresh.
+
+Revision ID: docker_012_drop_metrics
+Revises: docker_011_host_status
+Create Date: 2026-10-14
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_012_drop_metrics"
+down_revision: Union[str, None] = "docker_011_host_status"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS docker_metrics_hourly")
+    op.execute("DROP TABLE IF EXISTS docker_metrics")
+
+
+def downgrade() -> None:
+    op.create_table(
+        "docker_metrics",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("host_id", sa.String(36), sa.ForeignKey("hosts.id", ondelete="CASCADE"),
+                  nullable=False, index=True),
+        sa.Column("container_name", sa.String(255), nullable=False, index=True),
+        sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True),
+        sa.Column("cpu_pct", sa.Float, nullable=False),
+        sa.Column("mem_pct", sa.Float, nullable=False),
+        sa.Column("mem_usage_bytes", sa.BigInteger, nullable=False),
+    )
+    op.create_index("ix_docker_metrics_host_container_time", "docker_metrics",
+                    ["host_id", "container_name", "scraped_at"])
+    op.create_table(
+        "docker_metrics_hourly",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("host_id", sa.String(36), sa.ForeignKey("hosts.id", ondelete="CASCADE"),
+                  nullable=False),
+        sa.Column("container_name", sa.String(255), nullable=False),
+        sa.Column("bucket", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("cpu_pct", sa.Float, nullable=False),
+        sa.Column("mem_pct", sa.Float, nullable=False),
+        sa.Column("mem_usage_bytes", sa.BigInteger, nullable=False),
+        sa.Column("sample_count", sa.Integer, nullable=False),
+        sa.UniqueConstraint("host_id", "container_name", "bucket",
+                            name="uq_docker_metrics_hourly_bucket"),
+    )
+    op.create_index("ix_docker_metrics_hourly_bucket", "docker_metrics_hourly", ["bucket"])

plugins/docker/migrations/versions/docker_013_mem_used.py

@@ -0,0 +1,36 @@
+"""A container's memory is its processes' own, with file cache beside it
+
+Renames docker_containers.mem_usage_bytes / mem_pct to mem_used_bytes /
+mem_used_pct and adds mem_cache_bytes (#6030). The old figure was Docker's
+working set (usage less idle file cache), which still counted recently used
+file cache: a container that reads a lot of files showed hundreds of GB it
+didn't need. The new figure is anon + shmem, what the processes hold and
+can't hand back without swap, read by agent 1.23.0. The old values are
+cleared rather than carried under the new names; each container's next
+report from an updated agent fills them.
+
+Revision ID: docker_013_mem_used
+Revises: docker_012_drop_metrics
+Create Date: 2026-10-18
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_013_mem_used"
+down_revision: Union[str, None] = "docker_012_drop_metrics"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.alter_column("docker_containers", "mem_usage_bytes", new_column_name="mem_used_bytes")
+    op.alter_column("docker_containers", "mem_pct", new_column_name="mem_used_pct")
+    op.add_column("docker_containers", sa.Column("mem_cache_bytes", sa.BigInteger, nullable=True))
+    op.execute("UPDATE docker_containers SET mem_used_bytes = NULL, mem_used_pct = NULL")
+
+
+def downgrade() -> None:
+    op.drop_column("docker_containers", "mem_cache_bytes")
+    op.alter_column("docker_containers", "mem_used_pct", new_column_name="mem_pct")
+    op.alter_column("docker_containers", "mem_used_bytes", new_column_name="mem_usage_bytes")

plugins/docker/migrations/versions/docker_014_image_digests.py

@@ -0,0 +1,30 @@
+"""Which image a swarm service is pinned to, and which images each host holds
+
+Adds docker_swarm_services.image_digest (the sha256 digest a stack deploy
+pinned) and docker_disk_usage.digests_json (every "repo@sha256:…" the host
+holds), both from agent 1.26.0. A maintenance run spreads services back out
+only onto a node holding the service's image. Older agents leave them empty.
+
+Revision ID: docker_014_image_digests
+Revises: docker_013_mem_used
+Create Date: 2026-10-25
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_014_image_digests"
+down_revision: Union[str, None] = "docker_013_mem_used"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("docker_swarm_services", sa.Column("image_digest", sa.String(80), nullable=True))
+    op.add_column("docker_disk_usage", sa.Column("digests_json", sa.Text, nullable=False,
+                                                 server_default="[]"))
+
+
+def downgrade() -> None:
+    op.drop_column("docker_disk_usage", "digests_json")
+    op.drop_column("docker_swarm_services", "image_digest")

plugins/docker/models.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 from datetime import datetime, timezone
 from sqlalchemy import (
     BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text,
-    UniqueConstraint,
 )
 from sqlalchemy.orm import Mapped, mapped_column
 from steward.models.base import Base
@@ -30,10 +29,14 @@ class DockerContainer(Base):
     # running | stopped | paused | exited | dead
     cpu_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
     # BigInteger: a container's memory usage/limit routinely exceeds 2^31 bytes
-    # (~2.1 GB), which overflows int4 and fails the insert.
+    # (~2.1 GB), which overflows int4 and fails the insert. Used is what its
-    mem_usage_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+    # processes hold (anon + shmem); file cache is kept apart, since the kernel
+    # gives it back when the host needs it (#6030). used_pct is of the limit,
+    # which is the host's RAM when none is set.
+    mem_used_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
     mem_limit_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
-    mem_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
+    mem_used_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
+    mem_cache_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
     restart_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     ports_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
     # JSON: [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}]
@@ -61,69 +64,6 @@ class DockerContainer(Base):
     blk_write_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
 
 
-class DockerMetric(Base):
-    """Time-series CPU/memory per container — one row per sample per running
-    container, scoped to the reporting host."""
-    __tablename__ = "docker_metrics"
-
-    id: Mapped[str] = mapped_column(
-        String(36), primary_key=True, default=time_ordered_id
-    )
-    host_id: Mapped[str] = mapped_column(
-        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
-    )
-    container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
-    scraped_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False,
-        default=lambda: datetime.now(timezone.utc),
-        index=True,
-    )
-    cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
-    mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
-    # BigInteger: per-container memory usage exceeds 2^31 bytes (~2.1 GB) routinely.
-    mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
-
-    # Per-container history lookups filter on (host_id, container_name) then sort
-    # by time — a composite index keeps the rows() sparkline queries cheap.
-    __table_args__ = (
-        Index("ix_docker_metrics_host_container_time",
-              "host_id", "container_name", "scraped_at"),
-    )
-
-
-class DockerMetricHourly(Base):
-    """Hourly rollup of docker_metrics — avg cpu/mem per container per hour.
-
-    Raw per-sample rows (~2880/container/day at 30s) are pruned beyond a short
-    window; before deletion they're aggregated here so multi-day history stays
-    cheap to store and query. One row per (host, container, hour bucket); the
-    unique constraint lets retention upsert idempotently if it re-runs before the
-    raw rows are deleted. `bucket` is the hour-truncated sample time.
-    """
-    __tablename__ = "docker_metrics_hourly"
-
-    id: Mapped[str] = mapped_column(
-        String(36), primary_key=True, default=time_ordered_id
-    )
-    host_id: Mapped[str] = mapped_column(
-        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
-    )
-    container_name: Mapped[str] = mapped_column(String(255), nullable=False)
-    bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
-    cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
-    mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
-    mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
-    sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-
-    __table_args__ = (
-        # One bucket per container per host — the conflict target for the
-        # idempotent rollup upsert; doubles as the history-query index.
-        UniqueConstraint("host_id", "container_name", "bucket",
-                         name="uq_docker_metrics_hourly_bucket"),
-        Index("ix_docker_metrics_hourly_bucket", "bucket"),
-    )
-
-
 class DockerEvent(Base):
     """Lifecycle events derived by diffing consecutive host snapshots.
 
@@ -167,7 +107,7 @@ class DockerLog(Base):
     """Container log lines pushed by the host agent — one row per line.
 
     Time-series, scoped to the reporting host (container names are only unique
-    within a host, same identity as docker_metrics). The agent tails each running
+    within a host, same identity as docker_containers). The agent tails each running
     container incrementally and folds new lines into its metrics push; `ts` is the
     line's own Docker timestamp. Bounded by a per-container size+age ring
     (retention), so a chatty container just keeps a shorter window rather than
@@ -219,6 +159,9 @@ class DockerSwarmService(Base):
     desired: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     running: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     image: Mapped[str | None] = mapped_column(String(512), nullable=True)
+    # The digest a stack deploy pinned ("sha256:…"), agent ≥ 1.26.0; None when
+    # the spec names a tag only.
+    image_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
     # task→node placement of running replicas: [{"node_id", "running"}], JSON.
     # A manager sees every task, so this captures cross-node placement that the
     # local docker_containers rows (this host only) can't.
@@ -256,6 +199,10 @@ class DockerDiskUsage(Base):
     images_active: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     containers_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     volumes_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    # Every image the host holds by repository digest, ["repo@sha256:…"], JSON
+    # (agent ≥ 1.26.0): what a swarm service's pinned image is checked against
+    # before a maintenance run moves the service here.
+    digests_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
     updated_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False,
         default=lambda: datetime.now(timezone.utc),

plugins/docker/overview.py

@@ -0,0 +1,150 @@
+# plugins/docker/overview.py
+"""The Docker page's two tables, as plain rows (#5299, #5301): one row per
+swarm service and one per standalone container, with the search, stack, host
+and status filters and the column sort the page offers. Model-free, so it is
+unit-tested without a database; routes.rows() feeds it.
+
+A stack is how the operator thinks of a group of services. Swarm names a
+stack's services "<stack>_<service>", so a service's stack is the part of its
+name before the first "_"; a standalone container's is its compose project.
+Stacks filter the list rather than section it, as host groups do on Hosts.
+"""
+from __future__ import annotations
+
+from datetime import datetime
+
+from .swarm_view import service_trouble
+
+# Status filter values and their labels; NO_STACK picks rows with no stack.
+DOCKER_STATES = {"problem": "Needs attention", "healthy": "Healthy", "stopped": "Stopped"}
+NO_STACK = "__none__"
+# Each sort and whether it starts descending: the busiest first; "up" starts
+# with the shortest uptime, the thing that most recently restarted.
+DOCKER_SORTS = {"status": False, "name": False, "stack": False, "cpu": True, "mem": True,
+                "up": False}
+
+
+def stack_of(service_name: str) -> str | None:
+    """The stack a swarm service was deployed in: "arr" for "arr_radarr";
+    None for a name with no "_" (a service created on its own)."""
+    stack, sep, rest = service_name.partition("_")
+    return stack if sep and stack and rest else None
+
+
+def _seconds_up(started_at, now: datetime) -> float | None:
+    return None if started_at is None else max((now - started_at).total_seconds(), 0.0)
+
+
+def service_row(s: dict, now: datetime) -> dict:
+    """A built swarm service (swarm_view.build_swarm_services) as a table row.
+    CPU and memory add up the running replicas; "up" is the newest replica's
+    start, so a service that just restarted a task reads as just restarted."""
+    reals = [r for r in s["replicas"] if not r["ghost"]]
+    running = [r for r in reals if (r["status"] or "").lower() == "running"]
+    cpu = [r["cpu_pct"] for r in running if r.get("cpu_pct") is not None]
+    mem = [r["mem_bytes"] for r in running if r.get("mem_bytes") is not None]
+    starts = [r["started_at"] for r in running if r.get("started_at")]
+    started = max(starts) if starts else None
+    return {
+        "kind": "service", "name": s["name"], "stack": stack_of(s["name"]),
+        "mode": s["mode"],
+        "image": s.get("image") or next((r["image"] for r in reals if r.get("image")), None),
+        "running": s["running"], "desired": s["desired"], "state": s["state"],
+        "trouble": service_trouble(s),
+        "nodes": sorted({r["host"] for r in reals}, key=str.lower),
+        "ghost_nodes": sorted({r["host"] for r in s["replicas"] if r["ghost"]}, key=str.lower),
+        "cpu_cores": sum(cpu) / 100 if cpu else None,
+        "mem_bytes": sum(mem) if mem else None,
+        "started_at": started, "up_s": _seconds_up(started, now),
+        "unhealthy": any(r.get("health") == "unhealthy" for r in reals),
+        "failures_24h": s.get("failures_24h"), "crash_looping": s.get("crash_looping"),
+        "last_error": s.get("last_error"), "trouble_since": s.get("trouble_since"),
+    }
+
+
+def container_trouble(c, *, stale: bool, exit_failed: bool) -> int | None:
+    """How bad a standalone container is, on service_trouble's scale: 1
+    unhealthy or restarting, 2 died with a failing exit, 3 its report is
+    stale; None when nothing is wrong (a cleanly stopped one included)."""
+    if c.health == "unhealthy" or c.status == "restarting":
+        return 1
+    if c.status != "running" and exit_failed:
+        return 2
+    if stale:
+        return 3
+    return None
+
+
+def container_row(c, *, label: str, host: str, usage: dict, ports: list, stale: bool,
+                  exit_failed: bool, now: datetime) -> dict:
+    """A standalone container as a table row; `usage` is service.usage()."""
+    running = c.status == "running"
+    return {
+        "kind": "container", "name": label, "container_name": c.name, "host": host,
+        "host_id": c.host_id, "stack": c.compose_project or None, "image": c.image,
+        "status": c.status, "health": c.health, "running": running, "ports": ports,
+        "usage": usage, "cpu_cores": usage.get("cores") if running else None,
+        "mem_bytes": usage.get("mem_bytes") if running else None,
+        "started_at": c.started_at if running else None,
+        "up_s": _seconds_up(c.started_at, now) if running else None,
+        "exit_code": c.exit_code, "oom_killed": c.oom_killed, "exit_failed": exit_failed,
+        "restart_count": c.restart_count or 0, "stale": stale, "scraped_at": c.scraped_at,
+        "trouble": container_trouble(c, stale=stale, exit_failed=exit_failed),
+    }
+
+
+def _hosts(row: dict) -> list[str]:
+    return row["nodes"] + row["ghost_nodes"] if row["kind"] == "service" else [row["host"]]
+
+
+def _stopped(row: dict) -> bool:
+    return (row["running"] == 0) if row["kind"] == "service" else not row["running"]
+
+
+def filter_rows(rows: list[dict], *, q: str = "", stack: str = "", host: str = "",
+                state: str = "") -> list[dict]:
+    """Rows matching a search (name, container name, image or stack, any
+    case), a stack (or NO_STACK), a host or node, and a Status filter; an
+    empty value doesn't filter."""
+    q = (q or "").strip().lower()
+    out = []
+    for r in rows:
+        if q and not any(q in (v or "").lower() for v in
+                         (r["name"], r.get("container_name"), r["image"], r["stack"])):
+            continue
+        if (stack == NO_STACK and r["stack"]) or (stack and stack != NO_STACK and r["stack"] != stack):
+            continue
+        if host and host not in _hosts(r):
+            continue
+        if state == "problem" and r["trouble"] is None:
+            continue
+        if state == "healthy" and (r["trouble"] is not None or _stopped(r)):
+            continue
+        if state == "stopped" and not _stopped(r):
+            continue
+        out.append(r)
+    return out
+
+
+def sort_rows(rows: list[dict], sort: str, desc: bool) -> list[dict]:
+    """Rows in a column's order. Ties keep name order; a row with no value for
+    the column goes last whichever way it runs. "status" is problems first,
+    worst first, then running, then stopped."""
+    by_name = sorted(rows, key=lambda r: r["name"].lower())
+    if sort == "name":
+        return list(reversed(by_name)) if desc else by_name
+    if sort == "status":
+        return sorted(by_name, key=lambda r: (4 if r["trouble"] is None else r["trouble"],
+                                              _stopped(r)), reverse=desc)
+    value = {"stack": lambda r: (r["stack"] or "").lower() or None,
+             "cpu": lambda r: r["cpu_cores"], "mem": lambda r: r["mem_bytes"],
+             "up": lambda r: r["up_s"]}[sort]
+    have = [r for r in by_name if value(r) is not None]
+    return sorted(have, key=value, reverse=desc) + [r for r in by_name if value(r) is None]
+
+
+def filter_options(services: list[dict], containers: list[dict]) -> dict:
+    """The stack and host choices the filter bar offers, from every row."""
+    rows = services + containers
+    return {"stacks": sorted({r["stack"] for r in rows if r["stack"]}, key=str.lower),
+            "hosts": sorted({h for r in rows for h in _hosts(r)}, key=str.lower)}

plugins/docker/rekey.py

@@ -0,0 +1,198 @@
+# plugins/docker/rekey.py
+"""Moves history kept under a swarm task's own name, "<host>/<service>.<slot>.
+<task>", onto its service's key, "swarm/<service>": samples, hourly rollups,
+anomalies and the alert rules and maintenance windows that name it.
+
+The raw table holds millions of rows, too many to move in a migration that the
+app has to wait for. A small task moves what it can in a short pass in the
+background instead, newest history first, and does nothing once none are left. Network
+rates stay where they are: they are kept per host, and age out with retention.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from collections import Counter
+from typing import TYPE_CHECKING
+
+from sqlalchemy import delete, select, text, update
+
+from steward.core.scheduler import ScheduledTask
+from steward.models.alerts import AlertRule, MaintenanceWindow
+from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest
+from steward.models.norms import MetricAnomaly, MetricAnomalySpan
+from steward.models.tickets import TicketLink
+from steward.models.timeline import TimelineEntry
+from steward.timeline.record import anomaly_changed
+
+from .service import container_resource, task_service
+
+if TYPE_CHECKING:
+    from quart import Quart
+
+logger = logging.getLogger(__name__)
+
+BATCH = 1000            # raw samples moved in one go: each rewrites its row and every index on it
+EVERY_S = 15
+PASS_S = 2              # a key with only a few rows takes milliseconds, and there can be many
+REST = 4                # after a pass, wait this many times as long as it took, so ingest keeps the disk
+TRAFFIC = ("net_rx_bps", "net_tx_bps")
+_TASK_KEY = r"^[^/]+/.+\.([0-9]+|[a-z0-9]{25})\.[a-z0-9]{25}$"
+_NOT_TRAFFIC = "metric_name NOT IN ('net_rx_bps', 'net_tx_bps')"
+
+# Hours the service has no row for change name; hours it has are combined, as
+# two tasks of a service can share an hour.
+_RENAME_HOURS = text(f"""
+UPDATE plugin_metrics_hourly o SET resource_name = :new
+WHERE o.source_module = 'docker' AND o.resource_name = :old AND o.{_NOT_TRAFFIC}
+  AND NOT EXISTS (SELECT 1 FROM plugin_metrics_hourly n
+                  WHERE n.source_module = 'docker' AND n.resource_name = :new
+                    AND n.metric_name = o.metric_name AND n.bucket = o.bucket)
+""")
+_COMBINE_HOURS = text("""
+UPDATE plugin_metrics_hourly n SET
+    value_avg = COALESCE((n.value_avg * n.sample_count + o.value_avg * o.sample_count)
+                         / NULLIF(n.sample_count + o.sample_count, 0), n.value_avg),
+    value_max = GREATEST(n.value_max, o.value_max),
+    value_p95 = GREATEST(n.value_p95, o.value_p95),
+    sample_count = n.sample_count + o.sample_count
+FROM plugin_metrics_hourly o
+WHERE o.source_module = 'docker' AND o.resource_name = :old
+  AND n.source_module = 'docker' AND n.resource_name = :new
+  AND n.metric_name = o.metric_name AND n.bucket = o.bucket
+""")
+_DROP_HOURS = text(f"""
+DELETE FROM plugin_metrics_hourly
+WHERE source_module = 'docker' AND resource_name = :old AND {_NOT_TRAFFIC}
+""")
+
+
+def service_key(old: str) -> str:
+    """The key a task-named resource belongs under now."""
+    return container_resource("", old.split("/", 1)[1])
+
+
+async def old_keys(session) -> list[str]:
+    """Every task-named docker resource still holding something to move, to be
+    taken from the end: those with raw samples last, as they are what a chart
+    and a normal read first. The latest-value table stands for the raw samples,
+    as a key stays in it until its samples have aged out."""
+    async def names(column, *where) -> set[str]:
+        found = (await session.execute(select(column).distinct().where(*where, column.op("~")(_TASK_KEY)))).scalars()
+        return {k for k in found if task_service(k.split("/", 1)[1])}
+
+    recent = await names(PluginMetricLatest.resource_name, PluginMetricLatest.source_module == "docker",
+                         PluginMetricLatest.metric_name.notin_(TRAFFIC))
+    hourly = await names(PluginMetricHourly.resource_name, PluginMetricHourly.source_module == "docker",
+                         PluginMetricHourly.metric_name.notin_(TRAFFIC))
+    named = (await names(MetricAnomaly.resource_name, MetricAnomaly.source_module == "docker")
+             | await names(AlertRule.resource_name, AlertRule.source_module == "docker")
+             | await names(MaintenanceWindow.scope_resource, MaintenanceWindow.scope_source == "docker"))
+    logger.info("Swarm history under task names: %d keys with raw samples, %d with hourly rows, "
+                "%d named by anomalies, rules or windows", len(recent), len(hourly), len(named))
+    by_service = Counter(task_service(k.split("/", 1)[1]) for k in hourly - recent)
+    logger.info("Swarm history under task names, hourly only, most keys first: %s; for example %s",
+                by_service.most_common(10), sorted(hourly - recent)[:3])
+    return sorted(hourly - recent) + sorted((named - hourly) - recent) + sorted(recent)
+
+
+async def _merge_anomalies(session, new: str) -> None:
+    """One row per metric still to review, as for any key: the rows waiting on
+    the service become the newest of them, with all their occurrences."""
+    rows = (await session.execute(
+        select(MetricAnomaly).where(
+            MetricAnomaly.source_module == "docker", MetricAnomaly.resource_name == new,
+            MetricAnomaly.review == "to_review")
+        .order_by(MetricAnomaly.last_started_at.desc()))).scalars().all()
+    keep: dict[str, MetricAnomaly] = {}
+    for row in rows:
+        into = keep.setdefault(row.metric_name, row)
+        if into is row:
+            continue
+        await session.execute(update(MetricAnomalySpan).where(
+            MetricAnomalySpan.anomaly_id == row.id).values(anomaly_id=into.id))
+        into.occurrences += row.occurrences
+        into.started_at = min(into.started_at, row.started_at)
+        if row.ended_at is None:
+            into.ended_at = None
+        if row.peak_level > into.peak_level:
+            into.peak_level, into.peak_at = row.peak_level, row.peak_at
+            into.usual_hold, into.usual_peak = row.usual_hold, row.usual_peak
+        linked = (await session.execute(select(TicketLink.ticket_id).where(
+            TicketLink.ref_type == "metric_anomaly", TicketLink.ref_id == into.id))).scalars().all()
+        await session.execute(update(TicketLink).where(
+            TicketLink.ref_type == "metric_anomaly", TicketLink.ref_id == row.id,
+            TicketLink.ticket_id.notin_(linked)).values(ref_id=into.id))
+        await session.execute(delete(TicketLink).where(
+            TicketLink.ref_type == "metric_anomaly", TicketLink.ref_id == row.id))
+        await session.execute(delete(TimelineEntry).where(
+            TimelineEntry.ref_type == "metric_anomaly", TimelineEntry.ref_id == row.id))
+        await session.delete(row)
+
+
+async def move_key(session, old: str) -> bool:
+    """Move the next batch of `old`, and the rest of what it holds once its raw
+    samples are done. Returns whether it is finished. Runs inside the caller's
+    transaction."""
+    new = service_key(old)
+    batch = select(PluginMetric.id).where(
+        PluginMetric.source_module == "docker", PluginMetric.resource_name == old,
+        PluginMetric.metric_name.notin_(TRAFFIC)).limit(BATCH)
+    moved = await session.execute(
+        update(PluginMetric).where(PluginMetric.id.in_(batch)).values(resource_name=new)
+        .execution_options(synchronize_session=False))
+    if moved.rowcount >= BATCH:
+        return False
+    keys = {"old": old, "new": new}
+    for statement in (_RENAME_HOURS, _COMBINE_HOURS, _DROP_HOURS):
+        await session.execute(statement, keys)
+    await session.execute(delete(PluginMetricLatest).where(
+        PluginMetricLatest.source_module == "docker", PluginMetricLatest.resource_name == old,
+        PluginMetricLatest.metric_name.notin_(TRAFFIC)))
+    await session.execute(update(AlertRule).where(
+        AlertRule.source_module == "docker", AlertRule.resource_name == old).values(resource_name=new))
+    await session.execute(update(MaintenanceWindow).where(
+        MaintenanceWindow.scope_source == "docker", MaintenanceWindow.scope_resource == old)
+        .values(scope_resource=new))
+    renamed = await session.execute(update(MetricAnomaly).where(
+        MetricAnomaly.source_module == "docker", MetricAnomaly.resource_name == old)
+        .values(resource_name=new).execution_options(synchronize_session=False))
+    if not renamed.rowcount:
+        return True
+    session.expire_all()
+    await _merge_anomalies(session, new)
+    await session.flush()
+    for row in (await session.execute(select(MetricAnomaly).where(
+            MetricAnomaly.source_module == "docker", MetricAnomaly.resource_name == new))).scalars().all():
+        await anomaly_changed(session, row)
+    return True
+
+
+def make_rekey_task(app: "Quart") -> ScheduledTask:
+    remaining: list[str] | None = None
+    busy = asyncio.Lock()
+    resume_at = 0.0
+
+    async def rekey() -> None:
+        nonlocal remaining, resume_at
+        started = time.monotonic()
+        if remaining == [] or busy.locked() or started < resume_at:
+            return
+        async with busy, app.db_sessionmaker() as session:
+            if remaining is None:
+                async with session.begin():
+                    remaining = await old_keys(session)
+            deadline = time.monotonic() + PASS_S
+            while remaining and time.monotonic() < deadline:
+                async with session.begin():
+                    finished = await move_key(session, remaining[-1])
+                if not finished:        # a full batch of samples is enough for one pass
+                    break
+                remaining.pop()
+            if not remaining:
+                logger.info("Swarm history under task names: all moved")
+        now = time.monotonic()
+        resume_at = now + REST * (now - started)
+
+    return ScheduledTask(name="docker_rekey", coro_factory=rekey, interval_seconds=EVERY_S)

plugins/docker/retention.py

@@ -1,160 +1,95 @@
 # plugins/docker/retention.py
-"""Bound Docker time-series growth: roll up old metrics, prune old rows.
+"""Bound Docker history growth: prune old lifecycle events and log lines.
 
-Published as the "docker.run_retention" capability (see __init__.setup) so the
+Published as the "docker.run_retention" and "docker.prune_logs" capabilities
-core cleanup task can drive it WITHOUT importing the docker models (same
+(see __init__.setup) so the core cleanup task can drive them WITHOUT importing
-opportunistic-coupling pattern as docker.persist_host_samples). Runs inside the
+the docker models (same opportunistic-coupling pattern as
-caller's open transaction; never opens or commits its own.
+docker.persist_host_samples). Each runs inside the caller's open transaction;
+never opens or commits its own.
 
-The scaling concern is docker_metrics: ~2880 rows/container/day at a 30s sample.
+Container CPU and memory are not here: they live in the one metric store
-We keep raw samples for a short window, then aggregate everything older into
+(plugin_metrics) and its rollup and retention (#5752).
-hourly averages (docker_metrics_hourly) and delete the raw rows — so multi-day
-history stays cheap to store and query. docker_events is light but unbounded
-without a cutoff, so it gets a (longer) window too.
 """
 from __future__ import annotations
 
 from datetime import datetime, timedelta
 
 
-def _hour_floor(dt: datetime) -> datetime:
-    """Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
-    return dt.replace(minute=0, second=0, microsecond=0)
-
-
-def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
-    """Hour-aligned boundary below which raw metrics get rolled up + deleted.
-
-    Aligning to the hour means we only ever roll up *whole* elapsed hours — a
-    bucket is never split across the keep/roll boundary, so re-running can't
-    produce a partial-then-complete duplicate for the same hour.
-    """
-    return _hour_floor(now - timedelta(days=raw_days))
-
-
 async def run_docker_retention(
     session,
     *,
     events_days: int,
-    metrics_raw_days: int,
-    metrics_rollup_days: int,
-    logs_retention_days: int = 3,
-    logs_max_bytes_per_container: int = 5_000_000,
     now: datetime | None = None,
 ) -> dict:
-    """Roll up + prune Docker time-series. Returns a counts dict for logging.
+    """Prune docker_events older than the events window. Returns a counts
+    dict for logging. The container-log ring is prune_docker_logs, run in its
+    own transaction."""
+    from datetime import timezone
 
-    1. Aggregate docker_metrics older than the (hour-aligned) raw window into
+    from sqlalchemy import delete
-       docker_metrics_hourly (avg cpu/mem per container per hour), upserting so a
+
-       re-run is idempotent, then delete those raw rows.
+    from .models import DockerEvent
-    2. Prune rolled-up rows older than the rollup window.
+
-    3. Prune docker_events older than the events window.
+    if now is None:
-    4. Prune docker_logs with a per-container size+age ring (m79): drop lines
+        now = datetime.now(timezone.utc)
-       older than the age window, then keep only the newest ~cap bytes per
+    res = await session.execute(
-       (host, container).
+        delete(DockerEvent).where(DockerEvent.at < now - timedelta(days=events_days))
+    )
+    return {"events_pruned": res.rowcount or 0}
+
+
+async def prune_docker_logs(
+    session,
+    *,
+    retention_days: int,
+    max_bytes_per_container: int,
+    now: datetime | None = None,
+) -> dict:
+    """The docker_logs ring (m79): drop lines older than the age window, then
+    keep only the newest ~cap bytes per (host, container). Runs inside the
+    caller's open transaction, like run_docker_retention.
+
+    Only containers at or over the cap are ranked (#5449). Ranking every line
+    in the table each hour to find the few containers over it sorted the whole
+    table on slow storage.
     """
     from datetime import timezone
 
-    from sqlalchemy import delete, func, select
+    from sqlalchemy import and_, delete, func, select
-    from sqlalchemy.dialects.postgresql import insert as pg_insert
 
-    from .models import DockerEvent, DockerLog, DockerMetric, DockerMetricHourly
+    from .models import DockerLog
 
     if now is None:
         now = datetime.now(timezone.utc)
 
-    rolled = rolled_rows = events_pruned = rollup_pruned = 0
-    logs_age_pruned = logs_size_pruned = 0
-
-    # ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
-    raw_cutoff = _rollup_cutoff(now, metrics_raw_days)
-    hour = func.date_trunc("hour", DockerMetric.scraped_at)
-    agg = (
-        select(
-            DockerMetric.host_id,
-            DockerMetric.container_name,
-            hour.label("bucket"),
-            func.avg(DockerMetric.cpu_pct).label("cpu_pct"),
-            func.avg(DockerMetric.mem_pct).label("mem_pct"),
-            func.avg(DockerMetric.mem_usage_bytes).label("mem_usage_bytes"),
-            func.count().label("sample_count"),
-        )
-        .where(DockerMetric.scraped_at < raw_cutoff)
-        .group_by(DockerMetric.host_id, DockerMetric.container_name, hour)
-    )
-    for r in (await session.execute(agg)).all():
-        stmt = (
-            pg_insert(DockerMetricHourly)
-            .values(
-                host_id=r.host_id,
-                container_name=r.container_name,
-                bucket=r.bucket,
-                cpu_pct=float(r.cpu_pct or 0.0),
-                mem_pct=float(r.mem_pct or 0.0),
-                mem_usage_bytes=int(r.mem_usage_bytes or 0),
-                sample_count=int(r.sample_count or 0),
-            )
-            .on_conflict_do_update(
-                constraint="uq_docker_metrics_hourly_bucket",
-                set_={
-                    "cpu_pct": float(r.cpu_pct or 0.0),
-                    "mem_pct": float(r.mem_pct or 0.0),
-                    "mem_usage_bytes": int(r.mem_usage_bytes or 0),
-                    "sample_count": int(r.sample_count or 0),
-                },
-            )
-        )
-        await session.execute(stmt)
-        rolled += 1
-        rolled_rows += int(r.sample_count or 0)
-    if rolled:
-        await session.execute(
-            delete(DockerMetric).where(DockerMetric.scraped_at < raw_cutoff)
-        )
-
-    # ── 2. Prune rolled-up rows beyond the rollup window ──
-    rollup_cutoff = now - timedelta(days=metrics_rollup_days)
     res = await session.execute(
-        delete(DockerMetricHourly).where(DockerMetricHourly.bucket < rollup_cutoff)
+        delete(DockerLog).where(DockerLog.ts < now - timedelta(days=retention_days))
     )
-    rollup_pruned = res.rowcount or 0
+    age_pruned = res.rowcount or 0
 
-    # ── 3. Prune lifecycle events beyond the events window ──
+    size = func.length(DockerLog.line)
-    events_cutoff = now - timedelta(days=events_days)
+    over = (
-    res = await session.execute(
+        select(DockerLog.host_id, DockerLog.container_name)
-        delete(DockerEvent).where(DockerEvent.at < events_cutoff)
+        .group_by(DockerLog.host_id, DockerLog.container_name)
+        .having(func.sum(size) >= max_bytes_per_container)
+        .subquery()
     )
-    events_pruned = res.rowcount or 0
+    # Per container, sum line bytes newest-first; delete a row once its
-
+    # strictly-newer siblings already fill the cap. The EXCLUSIVE prefix
-    # ── 4. Container-log ring: age cutoff, then per-container byte cap (m79) ──
+    # (running total minus this row) means the newest row always survives, so
-    logs_cutoff = now - timedelta(days=logs_retention_days)
+    # a single line larger than the cap is never wiped out.
-    res = await session.execute(
+    running = func.sum(size).over(
-        delete(DockerLog).where(DockerLog.ts < logs_cutoff)
-    )
-    logs_age_pruned = res.rowcount or 0
-
-    # Size ring: per (host, container), sum line bytes newest-first; delete a row
-    # once its strictly-newer siblings already fill the cap. Using the EXCLUSIVE
-    # prefix (running total minus this row) means the newest row always survives,
-    # so a single line larger than the cap is never wiped out.
-    running = func.sum(func.length(DockerLog.line)).over(
         partition_by=[DockerLog.host_id, DockerLog.container_name],
         order_by=[DockerLog.ts.desc(), DockerLog.id.desc()],
     )
-    prefix_excl = (running - func.length(DockerLog.line)).label("prefix_excl")
+    ranked = (
-    ranked = select(DockerLog.id, prefix_excl).subquery()
+        select(DockerLog.id, (running - size).label("prefix_excl"))
-    over_cap = select(ranked.c.id).where(
+        .join(over, and_(DockerLog.host_id == over.c.host_id,
-        ranked.c.prefix_excl >= logs_max_bytes_per_container)
+                         DockerLog.container_name == over.c.container_name))
-    res = await session.execute(
+        .subquery()
-        delete(DockerLog).where(DockerLog.id.in_(over_cap))
     )
-    logs_size_pruned = res.rowcount or 0
+    res = await session.execute(
-
+        delete(DockerLog).where(DockerLog.id.in_(
-    return {
+            select(ranked.c.id).where(ranked.c.prefix_excl >= max_bytes_per_container)))
-        "buckets_rolled": rolled,
+    )
-        "raw_rows_rolled": rolled_rows,
+    return {"logs_age_pruned": age_pruned, "logs_size_pruned": res.rowcount or 0}
-        "rollup_pruned": rollup_pruned,
-        "events_pruned": events_pruned,
-        "logs_age_pruned": logs_age_pruned,
-        "logs_size_pruned": logs_size_pruned,
-    }

plugins/docker/routes.py

@@ -12,6 +12,7 @@ from sqlalchemy import func, select
 from steward.auth.middleware import require_role
 from steward.models.users import UserRole
 from steward.models.hosts import Host
+from steward.core import anomalies as anom
 from steward.core.formatting import human_bytes, human_duration, seconds_since
 from steward.core.time_range import parse_range, DEFAULT_RANGE
 from . import service as docker_svc
@@ -20,8 +21,13 @@ from .swarm_view import (
     build_swarm_services, failing_services, parse_tasks, replaced_tasks, services_problems_first,
     task_summary, trouble_since,
 )
+from .swarm_nodes import node_cards
+from .overview import (
+    DOCKER_SORTS, DOCKER_STATES, NO_STACK, container_row, filter_rows, service_row, sort_rows,
+    stack_of,
+)
 from .models import (
-    DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage, DockerMetric,
+    DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage,
     DockerSwarmNode, DockerSwarmService,
 )
 
@@ -41,25 +47,35 @@ def _uptime(started_at: datetime | None) -> str | None:
     return None if secs is None or secs < 0 else human_duration(secs)
 
 
-def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
+def _filters(args) -> dict:
-    if len(values) < 2:
+    """The Docker page's search, filters and sort from the query string
-        return f'<svg width="{width}" height="{height}"></svg>'
+    (#5299): q, stack, host, state, sort, dir; anything unknown is dropped."""
-    mn, mx = min(values), max(values)
+    sort = args.get("sort") if args.get("sort") in DOCKER_SORTS else "status"
-    if mx == mn:
+    desc = {"desc": True, "asc": False}.get(args.get("dir"), DOCKER_SORTS[sort])
-        mx = mn + 1.0
+    return {"q": (args.get("q") or "").strip(), "stack": args.get("stack", ""),
-    step = width / (len(values) - 1)
+            "host": args.get("host", ""),
-    pts = []
+            "state": args.get("state") if args.get("state") in DOCKER_STATES else "",
-    for i, v in enumerate(values):
+            "sort": sort, "dir": "desc" if desc else "asc"}
-        x = i * step
+
-        y = height - (v - mn) / (mx - mn) * (height - 2) - 1
+
-        pts.append(f"{x:.1f},{y:.1f}")
+def _page_url(f: dict, **change) -> str:
-    poly = " ".join(pts)
+    """The Docker page with some filters changed; defaults are left out."""
-    return (
+    from urllib.parse import urlencode
-        f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
+    want = {**f, **change}
-        f'style="vertical-align:middle;">'
+    if want["dir"] == ("desc" if DOCKER_SORTS[want["sort"]] else "asc"):
-        f'<polyline points="{poly}" fill="none" stroke="#6060c0" stroke-width="1.5"/>'
+        want["dir"] = ""
-        f'</svg>'
+    if want["sort"] == "status":
-    )
+        want["sort"] = ""
+    query = urlencode({k: v for k, v in want.items() if v})
+    return "/plugins/docker/" + ("?" + query if query else "")
+
+
+def _sort_url(f: dict, key: str) -> str:
+    """A column header's link: its own default direction, or the other way
+    when it is already the sort."""
+    if key == f["sort"]:
+        return _page_url(f, sort=key, dir="asc" if f["dir"] == "desc" else "desc")
+    return _page_url(f, sort=key, dir="desc" if DOCKER_SORTS[key] else "asc")
 
 
 @docker_bp.get("/")
@@ -73,21 +89,64 @@ async def index():
             select(DockerSwarmService.host_id).limit(1))).first() is not None
         has_disk = (await db.execute(
             select(DockerDiskUsage.host_id).limit(1))).first() is not None
+        choices = await _filter_choices(db)
+        # One line per host under the tables: what Docker keeps on its disk and
+        # how much that grew in a week (#5306); the Disk page has the rest.
+        disk = await docker_svc.disk_hosts(
+            db, datetime.now(timezone.utc) - timedelta(days=7), points=14) if has_disk else []
     return await render_template(
         "docker/index.html",
         poll_interval=poll_interval,
         has_swarm=has_swarm,
-        has_disk=has_disk,
+        has_disk=has_disk, disk=disk,
+        f=_filters(request.args), choices=choices, states=DOCKER_STATES, no_stack=NO_STACK,
     )
 
 
+async def _filter_choices(db) -> dict:
+    """The stacks and hosts the filter bar offers: every swarm service's stack
+    and compose project, and every host or node that runs something."""
+    names = set((await db.execute(select(DockerSwarmService.service_name).distinct())).scalars())
+    names |= {n for n in (await db.execute(
+        select(DockerContainer.service_name).distinct())).scalars() if n}
+    stacks = {stack_of(n) for n in names} | {p for p in (await db.execute(
+        select(DockerContainer.compose_project).distinct())).scalars() if p}
+    host_ids = set((await db.execute(select(DockerContainer.host_id).distinct())).scalars())
+    hosts = {h.name for h in (await docker_svc.host_map(db, host_ids)).values()}
+    hosts |= {n for n in (await db.execute(select(DockerSwarmNode.hostname).distinct())).scalars() if n}
+    return {"stacks": sorted((s for s in stacks if s), key=str.lower),
+            "hosts": sorted(hosts, key=str.lower)}
+
+
+async def _swarm_context(db, now: datetime) -> dict:
+    """What the Docker page and a service's page both build from: containers,
+    swarm services and nodes, host names, and the swarm view (each service with
+    its current replicas, Swarm's task records and their failures)."""
+    containers = await docker_svc.list_containers(db)
+    services = list((await db.execute(select(DockerSwarmService))).scalars())
+    nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
+    hosts = await docker_svc.host_map(
+        db, {c.host_id for c in containers} | {n.host_id for n in nodes}
+        | {s.host_id for s in services})
+    node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
+    host_name = {h.id: h.name for h in hosts.values()}
+    swarm_view = build_swarm_services(
+        containers, services, node_hostname, host_name, now=now,
+        crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
+    return {"containers": containers, "services": services, "nodes": nodes, "hosts": hosts,
+            "node_hostname": node_hostname, "host_name": host_name, "swarm_view": swarm_view}
+
+
 @docker_bp.get("/rows")
 @require_role(UserRole.viewer)
 async def rows():
-    """HTMX fragment: what needs attention first, then swarm services
+    """HTMX fragment: a verdict line and the problems that aren't zero, what
-    (collapsed, cluster-complete), then non-swarm containers grouped by host."""
+    needs attention, what failed in 24h (each with its logs), then the swarm
+    services and the standalone containers as two tables that share the
+    filter bar's search, filters and sort (#5298–#5301)."""
     now = datetime.now(timezone.utc)
     stale_after = docker_svc.stale_after_seconds(current_app.config)
+    f = _filters(request.args)
 
     async with current_app.db_sessionmaker() as db:
         # Hosts whose Docker is not answering: their rows are what it last said.
@@ -95,84 +154,47 @@ async def rows():
             select(DockerHostStatus).where(DockerHostStatus.status != "ok"))).scalars()}
         ranked = await docker_svc.ranked_containers(
             db, now, docker_svc.crash_loop_deaths(current_app.config))
-        containers = await docker_svc.list_containers(db)
+        ctx = await _swarm_context(db, now)
-        services = list((await db.execute(select(DockerSwarmService))).scalars())
+        containers, hosts = ctx["containers"], ctx["hosts"]
-        nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
+        hosts.update(await docker_svc.host_map(db, set(docker_down) - set(hosts)))
-        hosts = await docker_svc.host_map(
-            db,
-            {c.host_id for c in containers}
-            | {n.host_id for n in nodes} | {s.host_id for s in services} | set(docker_down),
-        )
-
-        # Swarm tasks collapse into per-service rows (each replica labelled with
-        # its host, plus "ghost" replicas for tasks on agent-less nodes, drawn
-        # from the managers' placement). Non-swarm containers keep the host view.
-        node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
         host_name = {h.id: h.name for h in hosts.values()}
-        swarm_view = build_swarm_services(
+        swarm_view = ctx["swarm_view"]
-            containers, services, node_hostname, host_name,
-            crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
         capacity = await docker_svc.host_capacity(db, {c.host_id for c in containers})
+        norms = await docker_svc.container_norms(
+            db, {(c.host_id, c.name) for c in containers if not c.service_name}, now)
         failures = docker_svc.failures_24h(
             await docker_svc.failure_events(db, now - timedelta(hours=24)),
             swarm_view["services"], host_name)
 
+    thresholds = current_app.config.get("THRESHOLDS", {})
+    service_rows = [service_row(s, now) for s in swarm_view["services"]]
     standalone = [c for c in containers if not c.service_name]
-        groups: dict[str, dict] = {}
+    container_rows = [
-        for c in standalone:
+        container_row(
-            g = groups.get(c.host_id)
+            c, label=docker_svc.display_name(c), host=host_name.get(c.host_id, c.host_id),
-            if g is None:
+            usage=docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), thresholds=thresholds,
-                host = hosts.get(c.host_id)
+                                   norms=norms.get((c.host_id, c.name))),
-                g = groups[c.host_id] = {
+            ports=json.loads(c.ports_json) if c.ports_json else [],
-                    "host": host,
+            stale=_is_stale(c, now, stale_after, docker_down),
-                    "host_id": c.host_id,
+            exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed), now=now)
-                    "host_name": host.name if host else c.host_id,
+        for c in standalone]
-                    "containers": [], "running": 0, "stopped": 0,
+    desc = f["dir"] == "desc"
-                }
+    shown_services = sort_rows(filter_rows(service_rows, q=f["q"], stack=f["stack"], host=f["host"],
-            g["containers"].append({
+                                           state=f["state"]), f["sort"], desc)
-                "container": c,
+    shown_containers = sort_rows(filter_rows(container_rows, q=f["q"], stack=f["stack"],
-                "ports": json.loads(c.ports_json) if c.ports_json else [],
+                                             host=f["host"], state=f["state"]), f["sort"], desc)
-                "uptime": _uptime(c.started_at) if c.status == "running" else None,
-                "stale": _is_stale(c, now, stale_after, docker_down),
-                "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None))),
-                "exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
-            })
-            if c.status == "running":
-                g["running"] += 1
-            else:
-                g["stopped"] += 1
 
-    # Sub-group each host's containers by compose project, preserving the
+    failing = failing_services(swarm_view["services"])
-    # running-first ordering. Containers with no project fall into an unlabelled
-    # bucket rendered flat, so plain hosts look unchanged.
-    for g in groups.values():
-        subs: dict[str, dict] = {}
-        order: list[str] = []
-        for item in g["containers"]:
-            c = item["container"]
-            label = c.compose_project or None
-            key = label or ""
-            if key not in subs:
-                subs[key] = {"label": label, "containers": []}
-                order.append(key)
-            subs[key]["containers"].append(item)
-        g["subgroups"] = [subs[k] for k in order]
-        g["grouped"] = any(s["label"] for s in g["subgroups"])
-
-    host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
-    swarm_services = services_problems_first(swarm_view["services"])
-    standalone_running = sum(g["running"] for g in host_groups)
     attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down),
-                      label=docker_svc.task_label(r["container"], node_hostname))
+                      label=docker_svc.task_label(r["container"], ctx["node_hostname"]))
                  for r in docker_svc.needs_attention(ranked)]
-    failing = failing_services(swarm_services)
+    running = sum(r["running"] for r in container_rows)
     return await render_template(
         "docker/rows.html",
-        host_groups=host_groups,
+        services=shown_services, containers=shown_containers,
-        swarm_services=swarm_services,
+        services_total=len(service_rows), containers_total=len(container_rows),
-        failing_services=failing,
+        services_healthy=sum(s["state"] == "healthy" for s in service_rows),
-        failures=failures,
+        failing_services=failing, failures=failures,
-        services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
         docker_hosts=len({c.host_id for c in containers}),
         docker_down=sorted(
             ({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status,
@@ -181,9 +203,10 @@ async def rows():
         attention=attention, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
         unhealthy=sum("unhealthy" in r["problems"] for r in attention),
         restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
-        stale=sum(1 for g in host_groups for i in g["containers"] if i["stale"]),
+        stale=sum(r["stale"] for r in container_rows),
-        running=standalone_running,
+        running=running, stopped=len(container_rows) - running,
-        stopped=len(standalone) - standalone_running,
+        f=f, sort_url=lambda key: _sort_url(f, key), page_url=lambda **kw: _page_url(f, **kw),
+        filtered=any(f[k] for k in ("q", "stack", "host", "state")),
     )
 
 
@@ -214,7 +237,10 @@ async def widget():
             {c.host_id for c in all_containers}
             | {n.host_id for n in nodes} | {s.host_id for s in services},
         )
-        capacity = await docker_svc.host_capacity(db, {c.host_id for c in all_containers})
+        deaths = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
+        sick = await docker_svc.unhealthy_since(
+            db, [(c.host_id, c.name) for c in all_containers
+                 if not c.service_name and (c.health or "") == "unhealthy"])
 
     # Swarm tasks collapse into per-service rows (each replica tagged with its
     # host); non-swarm containers stay grouped by host.
@@ -223,46 +249,62 @@ async def widget():
     swarm_services = services_problems_first(build_swarm_services(
         all_containers, services, node_hostname, host_name,
         crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"])
-    failures = docker_svc.failures_24h(events, swarm_services, host_name)
+    # The dashboard shows what is wrong now: a service back to all its
+    # replicas has its failed tasks on the Docker page, not here (#5722).
+    still = {s["name"] for s in swarm_services if s["state"] != "healthy" or s.get("crash_looping")}
+    failures = [f for f in docker_svc.failures_24h(events, swarm_services, host_name)
+                if f["kind"] != "service" or f["name"] in still]
 
     standalone = [c for c in all_containers if not c.service_name]
     running = [c for c in standalone if c.status == "running"]
-    display = standalone if show_stopped else running
 
-    def trouble(c) -> int:
+    def troubled(c) -> bool:
-        if (c.status or "") == "restarting" or c.health == "unhealthy":
+        return ((c.status or "") == "restarting" or c.health == "unhealthy"
-            return 0
+                or (c.status != "running" and docker_svc.exit_is_failure(c.exit_code, c.oom_killed)))
-        if c.status != "running" and docker_svc.exit_is_failure(c.exit_code, c.oom_killed):
-            return 1
-        return 2 if c.status == "running" else 3
 
-    # Group for display so multi-host fleets read clearly; single-host stays flat.
+    # The widget-shape order (UI & Design ruling): what is wrong first, then
-    groups: dict[str, dict] = {}
+    # the healthy rest one line each, collapsing past HEALTHY_SHOWN.
-    for c in sorted(display, key=lambda c: (trouble(c), c.name)):
+    # Worst problem first, each with since when, as the Docker page has them.
-        g = groups.setdefault(c.host_id, {
+    hour_ago, crash = now - timedelta(hours=1), docker_svc.crash_loop_deaths(current_app.config)
-            "host_id": c.host_id,
+    trouble = {}
-            "host_name": host_name.get(c.host_id, c.host_id),
+    for c in standalone:
-            "containers": [],
+        if troubled(c):
-        })
+            died = deaths.get((c.host_id, c.name), [])
-        g["containers"].append({"c": c, "usage": docker_svc.usage(
+            problems = docker_svc.container_problems(
-            c, *capacity.get(c.host_id, (None, None))),
+                c, sum(1 for e in died if e.at >= hour_ago), bool(died), crash)
-            "exit_failed": docker_svc.exit_is_failure(c.exit_code, c.oom_killed)})
+            trouble[(c.host_id, c.name)] = (
-    host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
+                docker_svc.PROBLEMS.index(problems[0]) if problems else len(docker_svc.PROBLEMS),
+                docker_svc.problem_since(problems, died, hour_ago, sick.get((c.host_id, c.name))))
+    bad_containers = sorted((c for c in standalone if troubled(c)),
+                            key=lambda c: (trouble[(c.host_id, c.name)][0], c.name))
+    healthy = ([{"kind": "service", "s": s} for s in swarm_services if s["state"] == "healthy"]
+               + [{"kind": "container", "c": c} for c in sorted(running, key=lambda c: c.name)
+                  if not troubled(c)]
+               + ([{"kind": "stopped", "c": c} for c in sorted(standalone, key=lambda c: c.name)
+                   if c.status != "running" and not troubled(c)] if show_stopped else []))
 
     return await render_template(
         "docker/widget.html",
-        host_groups=host_groups,
+        bad_services=[s for s in swarm_services if s["state"] != "healthy"],
-        multi_host=len(host_groups) > 1,
+        bad_containers=bad_containers,
-        swarm_services=swarm_services,
+        bad_since={key: since for key, (_rank, since) in trouble.items()},
+        healthy=healthy[:HEALTHY_SHOWN], healthy_more=max(0, len(healthy) - HEALTHY_SHOWN),
+        services_total=len(swarm_services),
         services_healthy=sum(s["state"] == "healthy" for s in swarm_services),
         running_count=len(running),
         failures=failures,
+        host_name=host_name,
         total_count=len(all_containers) + len(swarm_services),
-        show_stopped=show_stopped,
         widget_id=widget_id,
     )
 
 
+# Healthy services and containers listed on the dashboard widget before the
+# rest collapse to "+N healthy"; the Docker page lists them all.
+HEALTHY_SHOWN = 40
+FAILURES_SHOWN = 5
+
+
 CPU_AVERAGE_MINUTES = 5
 
 
@@ -271,46 +313,68 @@ CPU_AVERAGE_MINUTES = 5
 async def widget_resources():
     """HTMX dashboard widget: the containers using the most CPU, averaged over
     the last few minutes so the order holds still between polls, with their
-    CPU in cores and memory in bytes."""
+    CPU in cores and memory in bytes, each beside bars of the last few hours
-    limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
+    scaled to the row's own peak (never below a floor, #5214)."""
+    limit = max(1, min(20, _arg_number("limit", 10, int)))
+    hours = max(1, min(24, _arg_number("hours", 6, int)))
+    cpu_floor = max(0.1, _arg_number("cpu_floor", 1.0, float))
+    mem_floor = max(0.1, _arg_number("mem_floor", 1.0, float))
     widget_id = request.args.get("wid", "0")
-    since = datetime.now(timezone.utc) - timedelta(minutes=CPU_AVERAGE_MINUTES)
+    now = datetime.now(timezone.utc)
+    since = now - timedelta(minutes=CPU_AVERAGE_MINUTES)
+    slots, width = docker_svc.bar_slots(now, hours)
 
     async with current_app.db_sessionmaker() as db:
         # Dedup before the limit, else swarm tasks reported by both managers can
         # fill the list with duplicates of the same container.
         running = dedup_by_container_id([
             c for c in await docker_svc.list_containers(db) if c.status == "running"])
-        average = {(h, n): v for h, n, v in (await db.execute(
+        average = await docker_svc.average_cpu(db, {(c.host_id, c.name) for c in running}, since)
-            select(DockerMetric.host_id, DockerMetric.container_name, func.avg(DockerMetric.cpu_pct))
-            .where(DockerMetric.scraped_at >= since)
-            .group_by(DockerMetric.host_id, DockerMetric.container_name)
-        )).all()}
-        nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
-        hosts = await docker_svc.host_map(db, {c.host_id for c in running})
-        capacity = await docker_svc.host_capacity(db, {c.host_id for c in running})
 
         def cpu_of(c):
             v = average.get((c.host_id, c.name))
             return c.cpu_pct if v is None else float(v)
 
         busiest = sorted(running, key=lambda c: (-(cpu_of(c) or 0), c.name))[:limit]
-    node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
+        history = await docker_svc.recent_usage(
+            db, {(c.host_id, c.name) for c in busiest}, slots[0], width)
+        hosts = await docker_svc.host_map(db, {c.host_id for c in busiest})
+        capacity = await docker_svc.host_capacity(db, {c.host_id for c in busiest})
+        norms = await docker_svc.container_norms(db, {(c.host_id, c.name) for c in busiest})
+    thresholds = current_app.config.get("THRESHOLDS", {})
+
+    def bars(c, i, scale, floor):
+        seen = history.get((c.host_id, c.name), {})
+        values = [None if seen.get(t) is None or seen[t][i] is None else seen[t][i] * scale
+                  for t in slots]
+        return docker_svc.scaled_bars(values, floor)
+
     rows_data = [{
         "c": c,
-        "label": docker_svc.task_label(c, node_hostname),
+        "label": docker_svc.display_name(c),
         "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id,
-        "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c)),
+        "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c),
+                                  thresholds=thresholds, norms=norms.get((c.host_id, c.name))),
+        "cpu_bars": bars(c, 0, 0.01, cpu_floor),            # cpu_pct → cores
+        "mem_bars": bars(c, 1, 1.0, mem_floor * 2**30),     # bytes; floor in GiB
     } for c in busiest]
     return await render_template(
         "docker/widget_resources.html",
         rows=rows_data,
         minutes=CPU_AVERAGE_MINUTES,
-        multi_host=len({c.host_id for c in busiest}) > 1,
+        hours=hours, cpu_floor=cpu_floor, mem_floor=mem_floor * 2**30,
         widget_id=widget_id,
     )
 
 
+def _arg_number(key, default, kind):
+    """A numeric query argument, or `default` when missing or not a number."""
+    try:
+        return kind(request.args.get(key, default))
+    except (TypeError, ValueError):
+        return default
+
+
 @docker_bp.get("/host/<host_id>")
 @require_role(UserRole.viewer)
 async def host_panel(host_id: str):
@@ -330,7 +394,9 @@ async def host_panel(host_id: str):
         nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
         failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
         capacity = await docker_svc.host_capacity(db, {host_id})
+        norms = await docker_svc.container_norms(db, {(c.host_id, c.name) for c in containers}, now)
 
+    thresholds = current_app.config.get("THRESHOLDS", {})
     replaced = replaced_tasks(containers, services)
     node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
     hour_ago = now - timedelta(hours=1)
@@ -345,37 +411,59 @@ async def host_panel(host_id: str):
         rows.append({
             "c": c, "label": docker_svc.task_label(c, node_hostname), "problems": problems,
             "since": docker_svc.problem_since(problems, events, hour_ago),
-            "usage": docker_svc.usage(c, *capacity.get(host_id, (None, None))),
+            "usage": docker_svc.usage(c, *capacity.get(host_id, (None, None)), thresholds=thresholds,
+                                      norms=norms.get((c.host_id, c.name))),
             "uptime": _uptime(c.started_at) if c.status == "running" else None,
         })
-    rows.sort(key=lambda r: (
+    shown, folded = host_panel_rows(rows)
-        docker_svc.PROBLEMS.index(r["problems"][0]) if r["problems"] else len(docker_svc.PROBLEMS),
-        r["c"].status != "running", r["label"].lower()))
     running = sum(1 for r in rows if r["c"].status == "running")
     return await render_template(
         "docker/host_panel.html",
-        rows=rows, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
+        rows=shown, folded=folded, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
         running=running, stopped=len(rows) - running,
         problems=sum(1 for r in rows if r["problems"]),
-        replaced=len(replaced),
+        replaced=len(replaced), busiest=HOST_PANEL_BUSIEST,
     )
 
 
-# Glyph + colour per lifecycle event, so the timeline reads at a glance.
+# Running containers shown before the rest fold away on a host's page (#5276).
+HOST_PANEL_BUSIEST = 8
+
+
+def host_panel_rows(rows: list) -> tuple[list, list]:
+    """(shown, folded) for a host's Docker panel: every container with a
+    problem, worst first; then the busiest running ones by CPU. The other
+    running containers (busiest first) and the stopped ones (by name) fold
+    away, so a host with dozens of containers keeps a short card."""
+    def cpu(r):
+        return getattr(r["c"], "cpu_pct", None) or 0.0
+    problems = sorted((r for r in rows if r["problems"]), key=lambda r: (
+        docker_svc.PROBLEMS.index(r["problems"][0]), r["label"].lower()))
+    running = sorted((r for r in rows if not r["problems"] and r["c"].status == "running"),
+                     key=lambda r: (-cpu(r), r["label"].lower()))
+    stopped = sorted((r for r in rows if not r["problems"] and r["c"].status != "running"),
+                     key=lambda r: r["label"].lower())
+    return (problems + running[:HOST_PANEL_BUSIEST],
+            running[HOST_PANEL_BUSIEST:] + stopped)
+
+
+# Glyph + tone per lifecycle event, so the timeline reads at a glance.
 _EVENT_STYLE = {
-    "start":         ("▲", "var(--green)"),
+    "start":         ("▲", "ok"),
-    "restart":       ("↻", "var(--yellow)"),
+    "restart":       ("↻", "warn"),
-    "stop":          ("■", "var(--text-muted)"),
+    "stop":          ("■", "muted"),
-    "die":           ("✕", "var(--red)"),
+    "die":           ("✕", "crit"),
-    "oom":           ("☠", "var(--red)"),
+    "oom":           ("☠", "crit"),
-    "health_change": ("◐", "var(--orange)"),
+    "health_change": ("◐", "warn"),
 }
 
 
 @docker_bp.get("/container/<host_id>/<name>")
 @require_role(UserRole.viewer)
 async def container_detail(host_id: str, name: str):
-    """Full detail page for one container: facts, resource history, lifecycle."""
+    """One container (#5310): a header that says whether it is well and what it
+    is, why it stopped only while it has, what it moved since it started, its
+    CPU and memory over time, and its lifecycle."""
     async with current_app.db_sessionmaker() as db:
         c = await db.get(DockerContainer, (host_id, name))
         host = await db.get(Host, host_id)
@@ -389,6 +477,13 @@ async def container_detail(host_id: str, name: str):
             )).scalar()
         cpus, mem_total = (await docker_svc.host_capacity(db, [host_id])).get(host_id, (None, None)) \
             if c is not None else (None, None)
+        # Its CPU and memory against their normal, and its anomalies (#5742).
+        norms = (await docker_svc.container_norms(db, {(host_id, name)})).get((host_id, name))
+        resource = docker_svc.container_resource(host.name, name) if host is not None else None
+        anomalies = await anom.views(db, await anom.list_anomalies(
+            db, source="docker", resource=resource,
+            since=datetime.now(timezone.utc) - timedelta(days=30), limit=100),
+            links=False) if resource else []
 
     if c is None:
         return await render_template(
@@ -404,7 +499,10 @@ async def container_detail(host_id: str, name: str):
         name=name,
         label=docker_svc.task_label(c, {c.node_id: node_name} if node_name else None),
         node_name=node_name,
-        usage=docker_svc.usage(c, cpus, mem_total),
+        usage=docker_svc.usage(c, cpus, mem_total, thresholds=current_app.config.get("THRESHOLDS", {}),
+                               norms=norms),
+        anomalies=anomalies, anomaly_back=f"/plugins/docker/container/{host_id}/{name}",
+        anomaly_resource=resource,
         exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed),
         ports=json.loads(c.ports_json) if c.ports_json else [],
         uptime=_uptime(c.started_at) if c.status == "running" else None,
@@ -412,10 +510,11 @@ async def container_detail(host_id: str, name: str):
         blk_read=human_bytes(c.blk_read_bytes), blk_write=human_bytes(c.blk_write_bytes),
         events=[
             {"event": e.event, "detail": e.detail, "at": e.at,
-             "glyph": _EVENT_STYLE.get(e.event, ("•", "var(--text-muted)"))[0],
+             "glyph": _EVENT_STYLE.get(e.event, ("•", "muted"))[0],
-             "colour": _EVENT_STYLE.get(e.event, ("•", "var(--text-muted)"))[1]}
+             "tone": _EVENT_STYLE.get(e.event, ("•", "muted"))[1]}
             for e in events
         ],
+        died_at=next((e.at for e in events if e.event in ("die", "oom")), None),
         current_range=request.args.get("range", DEFAULT_RANGE),
     )
 
@@ -423,7 +522,9 @@ async def container_detail(host_id: str, name: str):
 @docker_bp.get("/swarm")
 @require_role(UserRole.viewer)
 async def swarm():
-    """Swarm topology page: services with replica health, nodes, placement.
+    """The Swarm page, laid out by node (#5303): each node's role, whether it
+    takes work, what its containers use against what the machine has, and the
+    services placed on it; services short of replicas first.
 
     Swarm services/nodes are cluster-global — every manager's API returns the
     same list — but each manager reports them independently (rows keyed by
@@ -441,6 +542,20 @@ async def swarm():
                 DockerSwarmNode.role, DockerSwarmNode.hostname)
         )).scalars())
         hosts = await docker_svc.host_map(db, {s.host_id for s in services} | {n.host_id for n in nodes})
+        containers = await docker_svc.list_containers(db)
+        # The Steward host on each node: the one whose agent reports its
+        # containers, else the host named as the node is.
+        node_host = {c.node_id: c.host_id for c in containers if c.node_id}
+        unmatched = {(n.hostname or "").lower() for n in nodes if n.node_id not in node_host} - {""}
+        if unmatched:
+            by_name = {h.name.lower(): h for h in (await db.execute(
+                select(Host).where(func.lower(Host.name).in_(unmatched)))).scalars()}
+            for n in nodes:
+                h = by_name.get((n.hostname or "").lower())
+                if n.node_id not in node_host and h is not None:
+                    node_host[n.node_id] = h.id
+        hosts.update(await docker_svc.host_map(db, set(node_host.values()) - set(hosts)))
+        capacity = await docker_svc.host_capacity(db, set(node_host.values()))
 
     # ── Group reporting managers into swarms by shared node membership ──────────
     manager_ids = {s.host_id for s in services} | {n.host_id for n in nodes}
@@ -512,27 +627,33 @@ async def swarm():
                 **summary,
                 "trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
             })
-        # Problems first: a node that is down or not taking work, then by role.
-        node_rows = sorted(
-            node_by_id.values(), key=lambda n: (
-                n.status == "ready" and n.availability == "active",
-                n.role, (n.hostname or n.node_id).lower())
-        )
         manager_names = sorted(hosts[m].name if m in hosts else m for m in members)
         swarms.append({
             "managers": manager_names,
-            "services": services_problems_first(svc_dicts),
+            "services_total": len(svc_dicts),
-            "nodes": node_rows,
+            "services_healthy": sum(d["state"] == "healthy" for d in svc_dicts),
+            "troubled": failing_services(svc_dicts),
+            "nodes": node_cards(
+                node_by_id.values(), svc_by_name.values(), containers, node_host=node_host,
+                host_names={h.id: h.name for h in hosts.values()}, capacity=capacity,
+                service_state={d["name"]: d["state"] for d in svc_dicts}),
         })
 
     swarms.sort(key=lambda g: g["managers"][0].lower() if g["managers"] else "")
     return await render_template("docker/swarm.html", swarms=swarms)
 
 
+# How many images a host agent reports, largest first (host_agent agent.py
+# _disk_images caps its list here); a list this long may be missing smaller ones.
+IMAGES_REPORTED = 50
+
+
 @docker_bp.get("/disk")
 @require_role(UserRole.viewer)
 async def disk():
-    """Image/disk usage page: reclaimable space, per-image sizes, stopped count.
+    """What Docker keeps on each host's disk (#5304, #5305): the hosts side by
+    side with how much each grew over the selected range (the /disk/hosts
+    fragment), then each host's parts, cleanup and images by repository.
 
     Admins can reclaim space per host via the prune buttons, which fire the
     audited bundled prune playbook through Ansible (m78) — the collection agent
@@ -540,6 +661,7 @@ async def disk():
     """
     from steward.core.capabilities import has_capability
     from steward.models.ansible_inventory import AnsibleTarget
+    from .disk_view import group_images
 
     async with current_app.db_sessionmaker() as db:
         summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
@@ -569,39 +691,48 @@ async def disk():
     images_by_host: dict[str, list] = {}
     for im in images:
         images_by_host.setdefault(im.host_id, []).append({
-            "repo_tag": im.repo_tag, "size": human_bytes(im.size),
+            "repo_tag": im.repo_tag, "image_id": im.image_id, "size": im.size,
-            "shared": human_bytes(im.shared_size), "containers": im.containers,
+            "shared_size": im.shared_size, "containers": im.containers})
-            "reclaimable": im.containers == 0,
-        })
 
     host_groups = [
         {
             "host_id": s.host_id,
             "host_name": hosts[s.host_id].name if s.host_id in hosts else s.host_id,
-            "summary": {
+            "as_of": s.updated_at,
-                "images_total": s.images_total, "images_active": s.images_active,
+            "summary": s,
-                "images_size": human_bytes(s.images_size),
+            "total": s.images_size + s.containers_size + s.volumes_size + s.build_cache_size,
-                "images_reclaimable": human_bytes(s.images_reclaimable),
-                "layers_size": human_bytes(s.layers_size),
-                "containers_count": s.containers_count,
-                "containers_size": human_bytes(s.containers_size),
-                "volumes_count": s.volumes_count,
-                "volumes_size": human_bytes(s.volumes_size),
-                "build_cache_size": human_bytes(s.build_cache_size),
-            },
             "stopped": stopped_by_host.get(s.host_id, 0),
-            "images": images_by_host.get(s.host_id, []),
+            "repos": group_images(images_by_host.get(s.host_id, [])),
+            "images_listed": len(images_by_host.get(s.host_id, [])),
+            # The agent reports the largest IMAGES_REPORTED images (agent._disk_images).
+            "images_capped": len(images_by_host.get(s.host_id, [])) >= IMAGES_REPORTED,
             "has_target": s.host_id in target_by_host,
         }
         for s in summaries
     ]
-    host_groups.sort(key=lambda g: g["host_name"].lower())
+    host_groups.sort(key=lambda g: (-g["total"], g["host_name"].lower()))
     return await render_template(
         "docker/disk.html", host_groups=host_groups,
         ansible_available=has_capability("ansible.run_playbook"),
+        current_range=request.args.get("range", "7d"),
     )
 
 
+@docker_bp.get("/disk/hosts")
+@require_role(UserRole.viewer)
+async def disk_hosts_fragment():
+    """HTMX fragment: every Docker host's disk side by side, with how much its
+    total grew over the selected range and a line of it (#5304)."""
+    from steward.core.status import sparkline_svg
+    since, range_key = parse_range(request.args.get("range") or "7d")
+    async with current_app.db_sessionmaker() as db:
+        rows = await docker_svc.disk_hosts(db, since)
+    for r in rows:
+        r["spark"] = sparkline_svg([v for _, v in r["points"]], width=120, height=24,
+                                   stroke="currentColor")
+    return await render_template("docker/_disk_hosts.html", rows=rows, range_key=range_key)
+
+
 @docker_bp.post("/disk/<host_id>/prune")
 @require_role(UserRole.admin)
 async def disk_prune(host_id: str):
@@ -650,16 +781,35 @@ async def disk_prune(host_id: str):
 @docker_bp.get("/container/<host_id>/<name>/history")
 @require_role(UserRole.viewer)
 async def container_history(host_id: str, name: str):
-    """HTMX fragment: CPU + memory sparklines for the selected time range."""
+    """HTMX fragment: the container's CPU (cores) and memory over the
+    selected range, with the latest and the peak (#5310)."""
     since, range_key = parse_range(request.args.get("range"))
     async with current_app.db_sessionmaker() as db:
-        cpu_hist, mem_hist = await docker_svc.container_history(db, host_id, name, since)
+        series = await docker_svc.service_history(db, [(host_id, name)], since)
+    return await _usage_history(series, range_key)
+
+
+# A history line is drawn against at least this much, so a container idling
+# near zero draws flat (UI & Design ruling 8, #5310).
+HISTORY_CPU_FLOOR = 1.0           # cores
+HISTORY_MEM_FLOOR = 1 << 30       # bytes
+
+
+async def _usage_history(series: list, range_key: str):
+    """The CPU and memory history fragment both detail pages load: the latest
+    and the peak over a line that spans its half, scaled with a floor."""
+    from steward.core.status import sparkline_svg
+    cpu = [c / 100 for _, c, _ in series]
+    mem = [m for _, _, m in series]
     return await render_template(
-        "docker/_container_history.html",
+        "docker/_usage_history.html", have_data=len(series) >= 2, range_key=range_key,
-        sparkline_cpu=_sparkline(cpu_hist, width=320, height=48),
+        cpu_now=cpu[-1] if cpu else None, cpu_peak=max(cpu) if cpu else None,
-        sparkline_mem=_sparkline(mem_hist, width=320, height=48),
+        mem_now=mem[-1] if mem else None, mem_peak=max(mem) if mem else None,
-        have_data=len(cpu_hist) >= 2,
+        cpu_floor=HISTORY_CPU_FLOOR, mem_floor=HISTORY_MEM_FLOOR,
-        range_key=range_key,
+        spark_cpu=sparkline_svg(cpu, height=48, stroke="currentColor", fill_width=True,
+                                floor=HISTORY_CPU_FLOOR),
+        spark_mem=sparkline_svg(mem, height=48, stroke="currentColor", fill_width=True,
+                                floor=HISTORY_MEM_FLOOR),
     )
 
 
@@ -694,3 +844,154 @@ async def container_logs_lines(host_id: str, name: str):
         name=name, tail=_LOG_TAIL_DEFAULT,
         filtered=bool(query) or stream in ("stdout", "stderr"),
     )
+
+
+# ── A swarm service's own page (#5302) ───────────────────────────────────────
+
+def _task_label(slot: str, task_id: str) -> str:
+    """How a task reads beside its log lines: its slot and the start of its id."""
+    return f"{slot} · {task_id[:10]}"
+
+
+def _task_rows(tasks: list[dict], node_hostname: dict) -> list[dict]:
+    """Swarm's records of a service's newest tasks, newest first, as the page
+    lists them: where each ran, what state it reached, and why it ended."""
+    from .swarm_view import FAILED_TASK_STATES, task_time
+    out = []
+    for t in tasks:
+        nid = t.get("node_id") or ""
+        out.append({
+            "task_id": t.get("task_id") or "", "slot": str(t.get("slot") or ""), "node_id": nid,
+            "node": node_hostname.get(nid) or (nid[:12] if nid else "—"),
+            "state": t.get("state") or "unknown", "desired": t.get("desired_state") or "",
+            "exit_code": t.get("exit_code"), "why": t.get("error") or t.get("message") or "",
+            "at": task_time(t), "failed": t.get("state") in FAILED_TASK_STATES,
+            "current": t.get("state") == "running" and t.get("desired_state") == "running",
+        })
+    return out
+
+
+def _service_tasks(name: str, tasks: list[dict], replicas: list[dict],
+                   node_host: dict, host_name: dict) -> list[dict]:
+    """One table for a service's tasks (#5310): the running ones first, each
+    with what its container uses, then Swarm's record of the rest, newest
+    first. A node reads as its Steward host's name wherever one is known, so a
+    node is spelled one way down the table. A running replica Swarm's record
+    does not list (an agent before 1.8.0) still gets a row, and so does a node
+    running tasks with no Steward agent on it."""
+    live, unmatched = {}, []
+    for r in (r for r in replicas if not r["ghost"]):
+        slot, tid = docker_svc.task_of(name, r["name"]) or ("", "")
+        tid = r.get("task_id") or tid          # the agent's own record of it, else its name's
+        if tid:
+            live[tid] = (slot, r)
+        else:
+            unmatched.append(r)
+
+    def place(node_id, host_id, fallback):
+        hid = host_id or node_host.get(node_id)
+        return hid, (host_name.get(hid) if hid else None) or fallback
+
+    rows = []
+    for t in tasks:
+        _, r = live.pop(t["task_id"], (None, None))
+        hid, node = place(t["node_id"], r["host_id"] if r else None, t["node"])
+        rows.append(dict(t, node=node, host_id=hid, container=r["name"] if r else None,
+                         usage=r["usage"] if r else None, health=r["health"] if r else None,
+                         at=(r["started_at"] or t["at"]) if r and t["current"] else t["at"]))
+    for tid, (slot, r) in [*live.items(), *(("", ("", r)) for r in unmatched)]:
+        rows.append({"task_id": tid, "slot": slot, "node_id": "", "node": r["host"],
+                     "host_id": r["host_id"], "state": r["status"], "desired": "",
+                     "exit_code": None, "why": "", "at": r["started_at"], "failed": False,
+                     "current": (r["status"] or "").lower() == "running", "container": r["name"],
+                     "usage": r["usage"], "health": r["health"]})
+    listed = {t["node"] for t in tasks if t["current"]}
+    for g in (r for r in replicas if r["ghost"] and r["host"] not in listed):
+        rows.append({"task_id": "", "slot": "", "node_id": "", "node": g["host"], "host_id": None,
+                     "state": "running", "desired": "", "exit_code": None,
+                     "why": f"{g['count']} running on a node with no Steward agent: no figures",
+                     "at": None, "failed": False, "current": True, "container": None,
+                     "usage": None, "health": None})
+    return sorted(rows, key=lambda r: not r["current"])     # stable: newest first within each
+
+
+@docker_bp.get("/service/<name>")
+@require_role(UserRole.viewer)
+async def service_detail(name: str):
+    """One swarm service: where its replicas run and what they use, its image,
+    Swarm's record of its recent tasks, its CPU and memory over time across
+    every replica, and the logs of every task it has run — newest first, each
+    line marked with its task. `?task=` opens the logs on one task, which is
+    where a failure's "Logs →" lands."""
+    now = datetime.now(timezone.utc)
+    async with current_app.db_sessionmaker() as db:
+        ctx = await _swarm_context(db, now)
+        task_containers = await docker_svc.service_containers(db, name)
+        s = next((s for s in ctx["swarm_view"]["services"] if s["name"] == name), None)
+        capacity = await docker_svc.host_capacity(
+            db, {r["host_id"] for r in (s["replicas"] if s else []) if not r["ghost"]})
+        norms = await docker_svc.container_norms(
+            db, {(r["host_id"], r["name"]) for r in (s["replicas"] if s else []) if not r["ghost"]}, now)
+    if s is None and not task_containers:
+        return await render_template("docker/service.html", name=name, service=None), 404
+
+    newest = max((v for v in ctx["services"] if v.service_name == name),
+                 key=lambda v: v.updated_at, default=None)
+    tasks = _task_rows(parse_tasks(newest) if newest is not None else [], ctx["node_hostname"])
+    by_key = {(c.host_id, c.name): c for c in ctx["containers"]}
+    replicas = []
+    for r in (s["replicas"] if s else []):
+        c = None if r["ghost"] else by_key.get((r["host_id"], r["name"]))
+        replicas.append(dict(r, task_id=getattr(c, "task_id", None) if c is not None else None,
+                             usage=docker_svc.usage(c, *capacity.get(r["host_id"], (None, None)),
+                                                    thresholds=current_app.config.get("THRESHOLDS", {}),
+                                                    norms=norms.get((c.host_id, c.name)))
+                             if c is not None else None))
+    node_host = {c.node_id: c.host_id for c in ctx["containers"] if c.node_id}
+    table = _service_tasks(name, tasks, replicas, node_host, ctx["host_name"])
+    # Every task Steward holds logs or events for, newest known first.
+    known = {t["task_id"]: _task_label(t["slot"], t["task_id"]) for t in tasks if t["task_id"]}
+    for _, cname in task_containers:
+        slot, tid = docker_svc.task_of(name, cname)
+        known.setdefault(tid, _task_label(slot, tid))
+    focus = request.args.get("task", "")
+    return await render_template(
+        "docker/service.html", name=name, service=s,
+        row=service_row(s, now) if s else None, svc=newest, tasks=table,
+        task_choices=list(known.items()), focus_task=focus if focus in known else "",
+        stack=stack_of(name), current_range=request.args.get("range", DEFAULT_RANGE),
+    )
+
+
+@docker_bp.get("/service/<name>/history")
+@require_role(UserRole.viewer)
+async def service_history_fragment(name: str):
+    """HTMX fragment: the service's CPU (cores) and memory, summed across its
+    replicas, over the selected range, with the latest and the peak."""
+    since, range_key = parse_range(request.args.get("range"))
+    async with current_app.db_sessionmaker() as db:
+        series = await docker_svc.service_history(
+            db, await docker_svc.service_containers(db, name), since)
+    return await _usage_history(series, range_key)
+
+
+@docker_bp.get("/service/<name>/logs/lines")
+@require_role(UserRole.viewer)
+async def service_logs_lines(name: str):
+    """HTMX fragment: the newest retained lines across every task of the
+    service, stream, text and task filtered, each marked with its task."""
+    stream = request.args.get("stream", "all")
+    query = (request.args.get("q") or "").strip()
+    task = request.args.get("task") or None
+    async with current_app.db_sessionmaker() as db:
+        rows = await docker_svc.query_service_logs(
+            db, await docker_svc.service_containers(db, name), stream, query, _LOG_TAIL_DEFAULT,
+            task=task)
+    lines = []
+    for r in rows:
+        slot, tid = docker_svc.task_of(name, r.container_name) or ("?", r.container_name)
+        lines.append({"ts": r.ts, "stream": r.stream, "line": r.line, "task": _task_label(slot, tid)})
+    return await render_template(
+        "docker/_container_logs_lines.html", lines=lines, name=name, tail=_LOG_TAIL_DEFAULT,
+        filtered=bool(query) or bool(task) or stream in ("stdout", "stderr"), service=True,
+    )

plugins/docker/service.py

@@ -6,16 +6,19 @@ Quart's pipeline) call the same code.
 """
 from __future__ import annotations
 
+import re
 from datetime import datetime, timedelta, timezone
 
 from sqlalchemy import func, select
 
-from steward.core.time_range import bucket_seconds
+from steward.core import metric_series
+from steward.core.like import escape_like
 from steward.models.hosts import Host
+from steward.models.metrics import PluginMetric
 
 from .dedup import dedup_by_container_id
 from .models import (
-    DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly, DockerSwarmService,
+    DockerContainer, DockerEvent, DockerLog, DockerSwarmService,
 )
 from .swarm_view import replaced_tasks
 
@@ -72,7 +75,8 @@ def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
     (an agent older than 1.8.0) falls back to its tasks' deaths.
 
     Each row: {kind: "container"|"service", name, host_id, host, count, at,
-    detail}.
+    detail, task_id}; task_id is the failed swarm task, so its logs can be
+    found (None for a container, or a service without task records).
     """
     by_prefix = {s["name"] + ".": s for s in swarm_services}
     rows: dict[tuple, dict] = {}
@@ -96,7 +100,9 @@ def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
         if row is None:
             rows[key] = {"kind": kind, "name": name, "host_id": e.host_id,
                          "host": host_name.get(e.host_id, e.host_id), "count": 1, "at": e.at,
-                         "detail": "OOM-killed" if e.event == "oom" else e.detail}
+                         "detail": "OOM-killed" if e.event == "oom" else e.detail,
+                         "task_id": (task_of(svc["name"], e.container_name) or (None, None))[1]
+                         if svc else None}
         else:
             row["count"] += 1
     from .swarm_view import task_time
@@ -106,7 +112,8 @@ def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
             rows[("service", s["name"])] = {
                 "kind": "service", "name": s["name"], "host_id": None, "host": err.get("node"),
                 "count": s["failures_24h"], "at": task_time(err) if err else None,
-                "detail": err.get("error") or err.get("message") or err.get("state")}
+                "detail": err.get("error") or err.get("message") or err.get("state"),
+                "task_id": err.get("task_id")}
     epoch = datetime.min.replace(tzinfo=timezone.utc)
     return sorted(rows.values(), key=lambda r: r["at"] or epoch, reverse=True)
 
@@ -114,11 +121,11 @@ def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
 def task_label(c, node_hostname: dict | None = None) -> str:
     """How a container reads to a person. A swarm task is its service and
     slot: "codex_db · 1" for a replica, "agent · node-3" for a global
-    service's task on a node. Anything else is its own name."""
+    service's task on a node. Anything else as display_name reads it."""
     from .swarm_view import _slot_key
     svc = getattr(c, "service_name", None)
     if not svc:
-        return c.name
+        return display_name(c)
     slot = _slot_key(c)
     if slot.isdigit():
         return f"{svc} · {slot}"
@@ -126,28 +133,103 @@ def task_label(c, node_hostname: dict | None = None) -> str:
     return f"{svc} · {host}" if host else svc
 
 
-def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None) -> dict:
+_CI_JOB = re.compile(
+    r"^(?:GITEA|FORGEJO)-ACTIONS-TASK-\d+-WORKFLOW-(?P<workflow>.+?)-JOB-(?P<job>.+?)(?:-[0-9a-f]{16,})?$")
+# docker buildx's container driver names its node "buildx_buildkit_<builder><n>".
+_BUILDKIT = re.compile(r"^buildx_buildkit_(?P<builder>.+?)\d*$")
+
+
+def display_name(c) -> str:
+    """A container's name as a list can show it in one short line (#5253): a
+    swarm task by its service (the host says where it runs), a Gitea or
+    Forgejo Actions job container as "CI: <workflow> · <job>", a buildx
+    builder as "BuildKit builder · <name>", anything else by its own name."""
+    svc = getattr(c, "service_name", None)
+    if svc:
+        return svc
+    m = _CI_JOB.match(c.name or "")
+    if m:
+        return f"CI: {m['workflow']} · {m['job']}"
+    m = _BUILDKIT.match(c.name or "")
+    if m:
+        return f"BuildKit builder · {m['builder']}"
+    return c.name
+
+
+def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None, *,
+          thresholds: dict | None = None, norms: dict | None = None) -> dict:
     """A container's CPU and memory in terms a reader can weigh.
 
     Docker's CPU% is per core (100% = one core busy), so it is shown as cores
     and graded as a share of the host's CPUs when the host's core count is
-    known. Docker's memory limit is the host's RAM when none is set, so memory
+    known. Memory is what the container's processes hold, file cache left out
-    is shown in bytes, against a limit only when one is actually set; mem_pct
+    (#6030). Docker's memory limit is the host's RAM when none is set, so
-    is then a share of that limit, otherwise of the host's memory. `cpu_pct`
+    memory is shown in bytes, against a limit only when one is actually set;
+    mem_pct is then a share of that limit, otherwise of the host's memory. `cpu_pct`
     replaces the container's last sample (an average, say) when given.
+
+    With `thresholds`, cpu_grade and mem_grade are the grades to show: past
+    the cutoff but within the container's own normal for the hour (`norms`,
+    {metric: Norm}, container_norms) is "ok", and cpu_usual / mem_usual say
+    what is usual (#5742). Without, they are None and the page grades.
     """
     cpu = getattr(c, "cpu_pct", None) if cpu_pct is None else cpu_pct
     limit = getattr(c, "mem_limit_bytes", None)
     limited = bool(limit and mem_total and limit < mem_total * 0.95)
-    return {
+    out = {
         "cores": None if cpu is None else cpu / 100.0,
         "cpu_share": None if cpu is None or not cpus else cpu / cpus,
         "cpus": cpus,
-        "mem_bytes": getattr(c, "mem_usage_bytes", None),
+        "mem_bytes": getattr(c, "mem_used_bytes", None),
         "mem_limit": limit if limited else None,
-        "mem_pct": getattr(c, "mem_pct", None),
+        "mem_pct": getattr(c, "mem_used_pct", None),
         "mem_of": "its limit" if limited else ("the host's memory" if mem_total else None),
+        "cpu_grade": None, "cpu_usual": None, "mem_grade": None, "mem_usual": None,
     }
+    if thresholds is not None:
+        _grade(out, cpu, thresholds, norms or {})
+    return out
+
+
+# Container figures graded against their own normal (#5742): the metric that
+# learns it and its floor's setting (norms.floor.*).
+NORMED = {"cpu": ("cpu_pct", "container_cpu_pct"), "mem": ("mem_used_pct", "container_mem_pct")}
+
+
+def _grade(u: dict, cpu: float | None, thresholds: dict, norms: dict) -> None:
+    """Fill usage()'s grades. CPU is graded as a share of the host but learned
+    per core (Docker's own figure), so its normal is read in cores."""
+    from steward.core.anomalies import departs, graded
+    from steward.core.formatting import status_class
+    from steward.core.settings import floor_of
+    grade = status_class(u["cpu_share"], "cpu", thresholds)
+    norm = norms.get("cpu_pct")
+    if grade in ("warn", "crit") and norm is not None and cpu is not None \
+            and not departs(cpu, norm, floor_of(thresholds, "container_cpu_pct")):
+        grade = "ok"
+        u["cpu_usual"] = f"usual {norm.usual_hold / 100:.2f} cores at this hour"
+    u["cpu_grade"] = grade
+    if u["mem_of"]:
+        u["mem_grade"], u["mem_usual"] = graded(
+            u["mem_pct"], "mem", thresholds, norms.get("mem_used_pct"),
+            floor_of(thresholds, "container_mem_pct"), "%")
+    else:
+        u["mem_grade"] = "dim"
+
+
+async def container_norms(db, keys, now: datetime | None = None) -> dict:
+    """{(host_id, container_name): {metric: Norm}}: each container's CPU and
+    memory normals for this hour. One without enough history is absent."""
+    from steward.core.norms import norms_at
+    resources = await container_resources(db, keys)
+    got = await norms_at(db, [("docker", r, metric) for r in resources
+                              for metric, _f in NORMED.values()],
+                         now or datetime.now(timezone.utc))
+    out: dict = {}
+    for (_s, resource, metric), norm in got.items():
+        for key in resources[resource]:
+            out.setdefault(key, {})[metric] = norm
+    return out
 
 
 async def host_capacity(db, host_ids) -> dict[str, tuple[int | None, int | None]]:
@@ -185,38 +267,6 @@ async def host_capacity(db, host_ids) -> dict[str, tuple[int | None, int | None]
     return out
 
 
-def bucket(values: list[float], target: int = 40) -> list[float]:
-    """Bucket-average a series down to ~target points (DB-agnostic, in Python).
-
-    Replaces the old SQL strftime() bucketing, which was SQLite-only and broke
-    on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of
-    rows — averaging into a readable point count keeps the sparkline's shape.
-    """
-    n = len(values)
-    if n <= target:
-        return values
-    size = (n + target - 1) // target
-    out: list[float] = []
-    for i in range(0, n, size):
-        chunk = values[i:i + size]
-        out.append(sum(chunk) / len(chunk))
-    return out
-
-
-async def container_history(db, host_id: str, name: str, since) -> tuple[list, list]:
-    """Return (cpu_series, mem_series) sparkline-ready for one host's container."""
-    rows = (await db.execute(
-        select(DockerMetric.cpu_pct, DockerMetric.mem_pct)
-        .where(DockerMetric.host_id == host_id)
-        .where(DockerMetric.container_name == name)
-        .where(DockerMetric.scraped_at >= since)
-        .order_by(DockerMetric.scraped_at)
-    )).all()
-    cpu = bucket([r.cpu_pct or 0.0 for r in rows])
-    mem = bucket([r.mem_pct or 0.0 for r in rows])
-    return cpu, mem
-
-
 async def host_map(db, host_ids: set[str]) -> dict[str, Host]:
     if not host_ids:
         return {}
@@ -226,11 +276,6 @@ async def host_map(db, host_ids: set[str]) -> dict[str, Host]:
     }
 
 
-def escape_like(text: str) -> str:
-    """Make `text` match literally inside a LIKE/ILIKE pattern (escape char \\)."""
-    return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
-
-
 async def list_containers(db, host_id: str | None = None) -> list[DockerContainer]:
     """Containers, running first then by name; one host's when host_id is given."""
     stmt = select(DockerContainer)
@@ -271,48 +316,143 @@ async def query_logs(db, host_id: str, name: str, stream: str, query: str,
     return (await db.execute(stmt)).all()
 
 
-CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes")
+CONTAINER_SERIES = ("cpu_pct", "mem_used_pct", "mem_used_bytes", "mem_cache_bytes")
+
+
+# A swarm task's container: "<service>.<slot or node id>.<task id>".
+_TASK = re.compile(r"^(?P<service>.+?)\.(?:\d+|[a-z0-9]{25})\.[a-z0-9]{25}$")
+SERVICE_RESOURCE = "swarm/"
+
+
+def task_service(name: str) -> str | None:
+    """The swarm service a container is a task of, from its name, or None."""
+    m = _TASK.match(name or "")
+    return m["service"] if m else None
+
+
+# A container's series live in the one metric store: source "docker", recorded
+# at ingest. A swarm service's are under "swarm/<service>", which stays the same
+# when a task is replaced or moves host, and which all its tasks share. Any
+# other container's are under "<host name>/<container name>".
+def container_resource(host_name: str, name: str) -> str:
+    service = task_service(name)
+    return f"{SERVICE_RESOURCE}{service}" if service else f"{host_name}/{name}"
+
+
+def traffic_resource(host_name: str, name: str) -> str:
+    """Where a container's network rates are kept: "<host name>/<service or
+    container>", as they say who on that host is carrying its traffic."""
+    return f"{host_name}/{task_service(name) or name}"
+
+
+async def container_resources(db, keys) -> dict[str, list[tuple[str, str]]]:
+    """{resource: [(host_id, container_name), ...]} for (host_id,
+    container_name) keys whose host still exists. A swarm service's tasks share
+    one resource."""
+    keys = set(keys)
+    hosts = await host_map(db, {h for h, _ in keys})
+    out: dict[str, list[tuple[str, str]]] = {}
+    for h, n in keys:
+        if h in hosts:
+            out.setdefault(container_resource(hosts[h].name, n), []).append((h, n))
+    return out
 
 
 async def container_points(db, host_id: str, name: str, metric: str, since,
                            target: int = 120) -> list[tuple]:
-    """(time, average) points of one container metric since `since`, oldest first.
+    """(time, average) points of one container metric since `since`, oldest
-
+    first: raw samples bucket-averaged, and the hourly rollup for the part of
-    Raw samples are bucket-averaged in SQL; the part of the range older than
+    the range past the raw window (metric_series.points)."""
-    the oldest raw sample (already rolled up and pruned by retention) comes
-    from the hourly table.
-    """
     if metric not in CONTAINER_SERIES:
         raise ValueError(f"metric must be one of {', '.join(CONTAINER_SERIES)}")
-    raw_col = getattr(DockerMetric, metric)
+    host = await db.get(Host, host_id)
-    width_s = bucket_seconds(since, target)
+    if host is None:
-    rbucket = func.date_bin(
+        return []
-        func.make_interval(0, 0, 0, 0, 0, 0, width_s), DockerMetric.scraped_at, func.to_timestamp(0),
+    return await metric_series.points(db, "docker", container_resource(host.name, name),
-    ).label("bucket")
+                                      metric, since, target=target)
-    raw = (await db.execute(
+
-        select(rbucket, func.avg(raw_col))
+
-        .where(DockerMetric.host_id == host_id, DockerMetric.container_name == name,
+async def _samples(db, resources, metrics, since: datetime, width: int | None = None):
-               DockerMetric.scraped_at >= since)
+    """Rows of (resource, time, metric, value) for the given containers since
-        .group_by(rbucket).order_by(rbucket)
+    `since`: each sample, or with `width` (seconds) each bucket's average."""
-    )).all()
+    at = PluginMetric.recorded_at
-    oldest_raw = raw[0][0] if raw else None
+    if width:
-    hourly_stmt = (
+        at = func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, int(width)),
-        select(DockerMetricHourly.bucket, getattr(DockerMetricHourly, metric))
+                           PluginMetric.recorded_at, func.to_timestamp(0))
-        .where(DockerMetricHourly.host_id == host_id,
+    value = func.avg(PluginMetric.value) if width else PluginMetric.value
-               DockerMetricHourly.container_name == name,
+    q = (select(PluginMetric.resource_name, at.label("at"), PluginMetric.metric_name, value)
-               DockerMetricHourly.bucket >= since)
+         .where(PluginMetric.source_module == "docker",
-        .order_by(DockerMetricHourly.bucket)
+                PluginMetric.resource_name.in_(list(resources)),
-    )
+                PluginMetric.metric_name.in_(list(metrics)),
-    if oldest_raw is not None:
+                PluginMetric.recorded_at >= since))
-        hourly_stmt = hourly_stmt.where(DockerMetricHourly.bucket < oldest_raw)
+    if width:
-    hourly = (await db.execute(hourly_stmt)).all()
+        q = q.group_by(PluginMetric.resource_name, at, PluginMetric.metric_name)
-    return [(t, float(v)) for t, v in hourly] + [(t, float(v)) for t, v in raw]
+    return (await db.execute(q.order_by(at))).all()
+
+
+async def average_cpu(db, keys, since: datetime) -> dict[tuple[str, str], float]:
+    """{(host_id, container_name): its average cpu_pct since `since`}."""
+    resources = await container_resources(db, keys)
+    if not resources:
+        return {}
+    rows = (await db.execute(
+        select(PluginMetric.resource_name, func.avg(PluginMetric.value))
+        .where(PluginMetric.source_module == "docker",
+               PluginMetric.resource_name.in_(list(resources)),
+               PluginMetric.metric_name == "cpu_pct",
+               PluginMetric.recorded_at >= since)
+        .group_by(PluginMetric.resource_name))).all()
+    return {key: float(v) for r, v in rows if v is not None for key in resources[r]}
+
+
+# Busiest Containers bars (#5214): the recent past in BARS equal slices.
+BARS = 24
+
+
+def bar_slots(now: datetime, hours: int, bars: int = BARS) -> tuple[list[datetime], int]:
+    """The start of each of the last `bars` buckets covering `hours`, oldest
+    first, aligned as date_bin aligns them (to the epoch), and their width in
+    seconds. The newest bucket is the one `now` falls in."""
+    width = max(60, hours * 3600 // bars)
+    last = int(now.timestamp()) // width * width
+    return [datetime.fromtimestamp(last - (bars - 1 - i) * width, timezone.utc)
+            for i in range(bars)], width
+
+
+async def recent_usage(db, keys, since: datetime, width: int) -> dict:
+    """{(host_id, container_name): {bucket start: (cpu_pct, mem_used_bytes)}}
+    averaged per bucket since `since`, for the given containers, in one query."""
+    resources = await container_resources(db, keys)
+    if not resources:
+        return {}
+    out: dict = {}
+    for resource, at, metric, value in await _samples(
+            db, resources, ("cpu_pct", "mem_used_bytes"), since, width):
+        v = None if value is None else float(value)
+        for key in resources[resource]:
+            cpu, mem = out.setdefault(key, {}).get(at, (None, None))
+            out[key][at] = (v, mem) if metric == "cpu_pct" else (cpu, v)
+    return out
+
+
+def scaled_bars(values: list, floor: float) -> dict:
+    """Bar heights in % of a roof: the row's own peak, but never below `floor`,
+    so a container idling at 0.05 cores draws flat and its 0.05 → 0.10 wobble
+    doesn't look like a 1.5 → 3 core spike. None (no samples) stays a gap."""
+    present = [v for v in values if v is not None]
+    peak = max(present) if present else None
+    roof = max(peak or 0.0, floor)
+    return {"bars": [None if v is None else round(100.0 * v / roof, 1) for v in values],
+            "peak": peak, "roof": roof}
 
 
 async def search_logs(db, *, since, host_id: str | None = None, name: str | None = None,
+                      containers: list[tuple[str, str]] | None = None,
                       text: str = "", stream: str = "", before=None, limit: int = 50) -> list:
     """Retained log lines matching a filter, newest first (limit + 1 fetched by callers
-    that need to know whether more exist)."""
+    that need to know whether more exist). `containers` keeps the lines of those
+    (host_id, container_name) pairs: a swarm service's tasks (#5468)."""
+    from sqlalchemy import tuple_
     stmt = (select(DockerLog.id, DockerLog.host_id, DockerLog.container_name,
                    DockerLog.ts, DockerLog.stream, DockerLog.line)
             .where(DockerLog.ts >= since))
@@ -320,6 +460,10 @@ async def search_logs(db, *, since, host_id: str | None = None, name: str | None
         stmt = stmt.where(DockerLog.host_id == host_id)
     if name:
         stmt = stmt.where(DockerLog.container_name == name)
+    if containers is not None:
+        if not containers:
+            return []
+        stmt = stmt.where(tuple_(DockerLog.host_id, DockerLog.container_name).in_(containers))
     if stream in ("stdout", "stderr"):
         stmt = stmt.where(DockerLog.stream == stream)
     if text:
@@ -396,6 +540,93 @@ async def retained_names(db, want: str, *, exact: bool, host_id: str | None = No
     return rows[:limit]
 
 
+# A swarm task's container is named "<service>.<slot>.<task id>": slot is the
+# replica number, or the node id for a global service; task ids are 25 chars.
+_TASK_SUFFIX = re.compile(r"^(?P<slot>[A-Za-z0-9]+)\.(?P<task>[a-z0-9]{25})$")
+
+
+def task_of(service: str, container_name: str) -> tuple[str, str] | None:
+    """(slot, task id) when `container_name` is one of `service`'s task
+    containers, else None — so "arr" does not claim "arr_radarr.1.…"."""
+    if not container_name.startswith(service + "."):
+        return None
+    m = _TASK_SUFFIX.match(container_name[len(service) + 1:])
+    return (m["slot"], m["task"]) if m else None
+
+
+async def service_containers(db, service: str) -> list[tuple[str, str]]:
+    """(host_id, container_name) of every task container of a swarm service
+    that Steward still holds anything for: the live ones, and the ones Docker
+    removed that left events (each task container has a start event). The
+    page's logs and history read across all of them (#5302)."""
+    pattern = escape_like(service + ".") + "%"
+    found: set[tuple[str, str]] = set()
+    for model, col in ((DockerContainer, DockerContainer.name),
+                       (DockerEvent, DockerEvent.container_name)):
+        rows = (await db.execute(
+            select(model.host_id, col).where(col.like(pattern, escape="\\")).distinct())).all()
+        found.update((h, n) for h, n in rows if task_of(service, n))
+    return sorted(found)
+
+
+async def query_service_logs(db, containers, stream: str, query: str, limit: int,
+                             task: str | None = None) -> list:
+    """The newest retained lines across a service's task containers, newest
+    first, each with its container name so the page can say which task wrote
+    it. `task` keeps one task's lines (a failure's "Logs →")."""
+    from sqlalchemy import tuple_
+    pairs = [(h, n) for h, n in containers if task is None or n.endswith("." + task)]
+    if not pairs:
+        return []
+    stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line, DockerLog.container_name)
+            .where(tuple_(DockerLog.host_id, DockerLog.container_name).in_(pairs)))
+    if stream in ("stdout", "stderr"):
+        stmt = stmt.where(DockerLog.stream == stream)
+    if query:
+        stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\"))
+    stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
+    return (await db.execute(stmt)).all()
+
+
+def sum_series(samples, since: datetime, until: datetime, points: int = 60) -> list:
+    """[(bucket start, cpu %, memory bytes)] summed across containers: each
+    container's samples are averaged within a bucket, then the containers are
+    added, so two replicas read as their combined load. `samples` are
+    (container_name, at, cpu_pct, mem_bytes); buckets with no sample are left
+    out rather than read as zero."""
+    width = max((until - since).total_seconds() / points, 60.0)
+    acc: dict[int, dict[str, list]] = {}
+    for name, at, cpu, mem in samples:
+        i = int((at - since).total_seconds() // width)
+        if i < 0:
+            continue
+        per = acc.setdefault(i, {}).setdefault(name, [0.0, 0, 0])
+        per[0] += cpu or 0.0
+        per[1] += mem or 0
+        per[2] += 1
+    out = []
+    for i in sorted(acc):
+        cpu = sum(c / n for c, _, n in acc[i].values())
+        mem = sum(m / n for _, m, n in acc[i].values())
+        out.append((since + timedelta(seconds=i * width), cpu, int(mem)))
+    return out
+
+
+async def service_history(db, containers, since: datetime) -> list:
+    """sum_series over a service's task containers' samples since `since`."""
+    resources = await container_resources(db, containers)
+    if not resources:
+        return []
+    joined: dict[tuple[str, datetime], list] = {}
+    for resource, at, metric, value in await _samples(
+            db, resources, ("cpu_pct", "mem_used_bytes"), since):
+        pair = joined.setdefault((resource, at), [None, None])
+        pair[0 if metric == "cpu_pct" else 1] = value
+    rows = [(resource, at, cpu, mem) for (resource, at), (cpu, mem) in
+            sorted(joined.items(), key=lambda kv: kv[0][1])]
+    return sum_series(rows, since, datetime.now(timezone.utc))
+
+
 async def unhealthy_since(db, keys) -> dict[tuple[str, str], datetime]:
     """{(host_id, container_name): when it last turned unhealthy} for the given
     containers, from their health_change events; absent when none is retained."""
@@ -519,3 +750,25 @@ def stale_after_seconds(config) -> int:
     """How old a container's last report may be before the page calls it
     stale: the host agent's own staleness window, since the agent reports it."""
     return int(((config.get("PLUGINS") or {}).get("host_agent") or {}).get("stale_after_seconds", 180))
+
+
+async def disk_hosts(db, since: datetime, *, points: int = 60) -> list[dict]:
+    """Each Docker host's disk as `docker system df` last said, most space
+    first, with its recorded total since `since` (#5304): the points, oldest
+    first, and the growth over them (disk_view.growth; None until there are
+    two). Hosts are named by their Steward host."""
+    from steward.core import metric_series
+    from .disk_view import DISK_PARTS, growth
+    from .models import DockerDiskUsage
+    summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
+    hosts = await host_map(db, {s.host_id for s in summaries})
+    out = []
+    for s in summaries:
+        name = hosts[s.host_id].name if s.host_id in hosts else s.host_id
+        parts = {m: int(getattr(s, f) or 0) for m, f in DISK_PARTS.items()}
+        series = await metric_series.points(db, "docker", name, "disk_total_bytes", since, target=points)
+        out.append({"host_id": s.host_id, "host_name": name, "as_of": s.updated_at,
+                    "total": sum(parts.values()), "parts": parts,
+                    "reclaimable": int(s.images_reclaimable or 0), "points": series,
+                    "growth": growth(series)})
+    return sorted(out, key=lambda h: (-h["total"], h["host_name"].lower()))

plugins/docker/swarm_health.py

@@ -0,0 +1,111 @@
+"""The swarm's health as three figures an alert rule can watch (#5788).
+
+Every manager's agent reports the whole swarm, so the figures are swarm-wide,
+on one resource, `swarm`, and are recorded at most once every RECORD_EVERY
+whichever manager sent them: three managers then read as one series, and one
+manager going silent doesn't silence it.
+
+- swarm_services_short: services running fewer tasks than they want. A
+  service scaled to 0 wants none and is never short.
+- swarm_nodes_down: nodes whose status isn't ready, other than drained ones.
+  A drained node is one taken out of service on purpose (VDNT-Util01 always;
+  any node during its maintenance), so its going away is expected. Its host's
+  own "Agent offline" rule still covers it.
+- swarm_managers_unreachable: managers the swarm can't reach (agent 1.19.0
+  sends ManagerStatus.Reachability; before that, a manager whose status
+  isn't ready), drained ones aside for the same reason. With three managers
+  one can be lost and the swarm keeps working; two, and it can't change.
+
+A maintenance job on a swarm node holds these rules' notifications in its
+window (host_agent maintenance._swarm_window), since draining and rebooting a
+node is exactly what moves them.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+RESOURCE = "swarm"
+SWARM_METRICS = ("swarm_services_short", "swarm_nodes_down", "swarm_managers_unreachable")
+RECORD_EVERY = timedelta(seconds=30)
+
+_last_recorded: datetime | None = None
+
+
+def short_services(services: list[dict]) -> list[str]:
+    """The services swarm_services_short counts, each with how many of its
+    tasks run: "web_app (0 of 2 running)"."""
+    return [f"{s.get('service_name')} ({int(s.get('running') or 0)} of {int(s.get('desired') or 0)} running)"
+            for s in sorted(services, key=lambda s: s.get("service_name") or "")
+            if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0)]
+
+
+def _in_service(nodes: list[dict]) -> list[dict]:
+    return sorted((n for n in nodes if (n.get("availability") or "active") != "drain"),
+                  key=lambda n: n.get("hostname") or "")
+
+
+def down_nodes(nodes: list[dict]) -> list[str]:
+    """The nodes swarm_nodes_down counts, by hostname, with their status."""
+    return [f"{n.get('hostname') or n.get('node_id')} ({n.get('status') or 'unknown'})"
+            for n in _in_service(nodes) if (n.get("status") or "unknown") != "ready"]
+
+
+def unreachable_managers(nodes: list[dict]) -> list[str]:
+    """The managers swarm_managers_unreachable counts, by hostname."""
+    return [n.get("hostname") or n.get("node_id") for n in _in_service(nodes)
+            if n.get("role") == "manager"
+            and (n["reachability"] != "reachable" if n.get("reachability")
+                 else (n.get("status") or "unknown") != "ready")]
+
+
+def figures(swarm: dict) -> dict[str, float]:
+    """The three figures from one manager's swarm report."""
+    services = swarm.get("services") or []
+    nodes = swarm.get("nodes") or []
+    return {"swarm_services_short": float(len(short_services(services))),
+            "swarm_nodes_down": float(len(down_nodes(nodes))),
+            "swarm_managers_unreachable": float(len(unreachable_managers(nodes)))}
+
+
+async def _stored(session) -> tuple[list[dict], list[dict]]:
+    """The swarm's services and nodes as stored, the newest report of each
+    when more than one manager reported it, in the shape figures() reads.
+    A stored node has no reachability, so its status stands in for it."""
+    from sqlalchemy import select
+    from .mcp_tools import unique_nodes
+    from .models import DockerSwarmNode, DockerSwarmService
+    services: dict[str, DockerSwarmService] = {}
+    for s in (await session.execute(select(DockerSwarmService))).scalars():
+        if s.service_name not in services or s.updated_at > services[s.service_name].updated_at:
+            services[s.service_name] = s
+    nodes = unique_nodes(list((await session.execute(select(DockerSwarmNode))).scalars()))
+    return ([{"service_name": s.service_name, "desired": s.desired, "running": s.running}
+             for s in services.values()],
+            [{"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
+              "availability": n.availability, "status": n.status} for n in nodes])
+
+
+async def name_short_services(session, resource: str) -> list[str]:
+    return short_services((await _stored(session))[0])
+
+
+async def name_down_nodes(session, resource: str) -> list[str]:
+    return down_nodes((await _stored(session))[1])
+
+
+async def name_unreachable_managers(session, resource: str) -> list[str]:
+    return unreachable_managers((await _stored(session))[1])
+
+
+NAMERS = {"swarm_services_short": name_short_services, "swarm_nodes_down": name_down_nodes,
+          "swarm_managers_unreachable": name_unreachable_managers}
+
+
+def due(now: datetime) -> bool:
+    """Whether this report should be recorded: the first in RECORD_EVERY,
+    whichever manager it came from."""
+    global _last_recorded
+    if _last_recorded is not None and now - _last_recorded < RECORD_EVERY:
+        return False
+    _last_recorded = now
+    return True

plugins/docker/swarm_nodes.py

@@ -0,0 +1,71 @@
+# plugins/docker/swarm_nodes.py
+"""The Swarm page laid out by node (#5303): one card per node with its role,
+whether it takes work, what its containers use against what the machine has,
+and the services Swarm placed on it. Model-free, so it is unit-tested without
+a database; routes.swarm() feeds it.
+
+A node with a Steward agent reports its own containers, so its services carry
+CPU and memory. A node without one is known only from the managers' placement:
+its services are listed by count, with no figures.
+"""
+from __future__ import annotations
+
+from .swarm_view import _placement
+
+
+def node_ok(n) -> bool:
+    """A node that is up and taking work."""
+    return n.status == "ready" and n.availability == "active"
+
+
+def node_cards(nodes, services, containers, *, node_host: dict, host_names: dict,
+               capacity: dict, service_state: dict) -> list[dict]:
+    """One card per node, a node in trouble first, then managers, then by name.
+
+    `nodes` and `services` are one swarm's deduplicated rows; `containers`
+    every container row (only running swarm tasks on these nodes count);
+    `node_host` maps a node id to the Steward host on it, `capacity` a host id
+    to (cores, memory bytes), `service_state` a service name to healthy,
+    degraded or down."""
+    running_on: dict[str, dict[str, dict]] = {}
+    for c in containers:
+        if not (c.node_id and c.service_name and c.status == "running"):
+            continue
+        row = running_on.setdefault(c.node_id, {}).setdefault(
+            c.service_name, {"name": c.service_name, "tasks": 0, "cpu_pct": 0.0, "mem_bytes": 0,
+                             "measured": True})
+        row["tasks"] += 1
+        row["cpu_pct"] += c.cpu_pct or 0.0
+        row["mem_bytes"] += c.mem_used_bytes or 0
+    placed: dict[str, dict[str, int]] = {}
+    for s in services:
+        for p in _placement(s):
+            count = int(p.get("running") or 0)
+            if count > 0 and p.get("node_id"):
+                placed.setdefault(p["node_id"], {})[s.service_name] = count
+
+    cards = []
+    for n in nodes:
+        rows = dict(running_on.get(n.node_id, {}))
+        for name, count in placed.get(n.node_id, {}).items():
+            rows.setdefault(name, {"name": name, "tasks": count, "cpu_pct": None, "mem_bytes": None,
+                                   "measured": False})
+        for r in rows.values():
+            r["state"] = service_state.get(r["name"], "healthy")
+            r["cores"] = r["cpu_pct"] / 100 if r["cpu_pct"] is not None else None
+        tasks = sorted(rows.values(), key=lambda r: (r["state"] == "healthy", -(r["cores"] or 0.0),
+                                                     r["name"].lower()))
+        host_id = node_host.get(n.node_id)
+        cores, mem = capacity.get(host_id, (None, None)) if host_id else (None, None)
+        measured = [r for r in tasks if r["measured"]]
+        cards.append({
+            "node_id": n.node_id, "hostname": n.hostname or n.node_id[:12], "role": n.role,
+            "leader": n.leader, "status": n.status, "availability": n.availability, "ok": node_ok(n),
+            "host_id": host_id, "host_name": host_names.get(host_id) if host_id else None,
+            "agent": bool(running_on.get(n.node_id)) or host_id is not None,
+            "tasks": tasks, "task_count": sum(r["tasks"] for r in tasks),
+            "cores_used": sum(r["cores"] for r in measured) if measured else None,
+            "mem_used": sum(r["mem_bytes"] for r in measured) if measured else None,
+            "cores": cores, "mem": mem,
+        })
+    return sorted(cards, key=lambda c: (c["ok"], c["role"] != "manager", c["hostname"].lower()))

plugins/docker/swarm_view.py

@@ -94,7 +94,8 @@ def last_task_error(tasks: list[dict]) -> dict | None:
         if t.get("state") in FAILED_TASK_STATES or t.get("error"):
             return {"state": t.get("state"), "error": t.get("error") or None,
                     "message": t.get("message") or None, "exit_code": t.get("exit_code"),
-                    "node_id": t.get("node_id") or None, "at": t.get("at") or None}
+                    "node_id": t.get("node_id") or None, "at": t.get("at") or None,
+                    "task_id": t.get("task_id") or None, "slot": t.get("slot")}
     return None
 
 
@@ -233,11 +234,12 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
                          *, now: datetime | None = None, crash_loop_deaths: int = 3) -> dict:
     """Return {"services": [...], "standalone": [...], "superseded": {(host_id, name)}}.
 
-    services[i] = {name, mode, desired, running, state, replicas:[...],
+    services[i] = {name, mode, image, desired, running, state, replicas:[...],
                    previous_tasks, tasks_known, failures_1h, failures_24h,
                    crash_looping, last_error}
       real replica  = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
-                       health, restart_count}   # current tasks only
+                       mem_bytes, started_at, image, health, restart_count}
+                      # current tasks only
       ghost replica = {ghost:True, host, count}   # placement with no local row
       previous_tasks = task containers Swarm replaced, left out of replicas
       failures_*    = tasks that failed or were rejected in the window (None
@@ -287,7 +289,10 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
                 "name": c.name,
                 "status": getattr(c, "status", "unknown"),
                 "cpu_pct": getattr(c, "cpu_pct", None),
-                "mem_pct": getattr(c, "mem_pct", None),
+                "mem_pct": getattr(c, "mem_used_pct", None),
+                "mem_bytes": getattr(c, "mem_used_bytes", None),
+                "started_at": getattr(c, "started_at", None),
+                "image": getattr(c, "image", None) or None,
                 "health": getattr(c, "health", None),
                 "restart_count": getattr(c, "restart_count", 0) or 0,
             })
@@ -329,6 +334,7 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
         out_services.append({
             "name": name,
             "mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
+            "image": getattr(svc, "image", None) if svc is not None else None,
             "running": running,
             "desired": desired,
             "state": state,

plugins/docker/templates/docker/_container_history.html

@@ -1,17 +0,0 @@
-{# docker/_container_history.html — CPU/mem sparklines for the selected range #}
-{% if have_data %}
-<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:1.5rem;">
-  <div>
-    <div class="section-title" style="margin-bottom:0.4rem;">CPU %</div>
-    {{ sparkline_cpu | safe }}
-  </div>
-  <div>
-    <div class="section-title" style="margin-bottom:0.4rem;">Memory %</div>
-    {{ sparkline_mem | safe }}
-  </div>
-</div>
-{% else %}
-<div style="color:var(--text-muted);font-size:0.85rem;">
-  Not enough samples in this range yet — history fills in as the agent reports.
-</div>
-{% endif %}

plugins/docker/templates/docker/_container_logs_lines.html

@@ -1,9 +1,11 @@
 {# docker/_container_logs_lines.html — log-line fragment, HTMX-polled (m79).
-   {{ l.line }} is auto-escaped by Jinja, so log content can't inject markup. #}
+   {{ l.line }} is auto-escaped by Jinja, so log content can't inject markup.
+   A service's lines (#5302) carry the task each came from. #}
 {% if lines %}
 {% for l in lines %}
 <div style="display:flex;gap:0.6rem;white-space:pre-wrap;word-break:break-word;">
   <span style="color:var(--text-dim);flex-shrink:0;" title="{{ l.ts }}">{{ l.ts.strftime("%m-%d %H:%M:%S") }}</span>
+  {% if l.task %}<span class="tone-muted" style="flex-shrink:0;white-space:nowrap;" title="Slot and task id">{{ l.task }}</span>{% endif %}
   <span style="flex-shrink:0;width:3.2rem;color:{% if l.stream == 'stderr' %}var(--red){% else %}var(--text-muted){% endif %};">{{ l.stream }}</span>
   <span style="flex:1;min-width:0;{% if l.stream == 'stderr' %}color:var(--text);{% endif %}">{{ l.line }}</span>
 </div>
@@ -14,7 +16,7 @@
   No lines match the current filter.
   {% else %}
   No logs collected yet for <code>{{ name }}</code>. Lines appear within a few
-  seconds of the container writing to stdout/stderr — unless it's on the log
+  seconds of {{ "one of its tasks" if service else "the container" }} writing to stdout/stderr — unless it's on the log
   exclude list or log collection is turned off in
   <a href="/settings/thresholds/">Settings → Thresholds &amp; Retention</a>.
   {% endif %}

plugins/docker/templates/docker/_disk_hosts.html

@@ -0,0 +1,37 @@
+{# docker/_disk_hosts.html — every Docker host's disk side by side (#5304):
+   what it uses now, how much that grew over the selected range, and a line
+   of it; the parts after. A host is linked to its own section below. #}
+{% if rows %}
+<table class="table">
+  <thead><tr>
+    <th>Host</th>
+    <th class="num">Docker uses</th>
+    <th class="num" title="Change in the total over the last {{ range_key }}, from Steward's own record of it">Change ({{ range_key }})</th>
+    <th>Over {{ range_key }}</th>
+    <th class="num">Images</th><th class="num">Volumes</th><th class="num" title="Written by containers on top of their images">Container files</th>
+    <th class="num">Build cache</th><th class="num" title="Held by images no container uses">Reclaimable</th>
+  </tr></thead>
+  <tbody>
+    {% for r in rows %}
+    {% set g = r.growth %}
+    <tr>
+      <td><a href="#host-{{ r.host_id }}">{{ r.host_name }}</a></td>
+      <td class="num"><strong>{{ r.total|bytes }}</strong></td>
+      <td class="num">
+        {%- if g is none %}<span class="tone-dim" title="Steward records this from now on; growth shows once there are two readings">—</span>
+        {%- else %}<span class="tone-{{ 'warn' if g.pct is not none and g.pct >= 10 else ('ok' if g.change < 0 else 'muted') }}"
+              title="{{ g.from|bytes }} → {{ g.to|bytes }} since {{ g.since.strftime('%Y-%m-%d %H:%M UTC') }}">{{ "+" if g.change > 0 else ("−" if g.change < 0 else "±") }}{{ (g.change|abs)|bytes }}</span>{% endif -%}
+      </td>
+      <td class="tone-info">{{ r.spark|safe }}</td>
+      <td class="num">{{ r.parts.disk_images_bytes|bytes }}</td>
+      <td class="num">{{ r.parts.disk_volumes_bytes|bytes }}</td>
+      <td class="num">{{ r.parts.disk_containers_bytes|bytes }}</td>
+      <td class="num">{{ r.parts.disk_build_cache_bytes|bytes }}</td>
+      <td class="num">{% if r.reclaimable %}<span class="tone-warn">{{ r.reclaimable|bytes }}</span>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+    </tr>
+    {% endfor %}
+  </tbody>
+</table>
+{% else %}
+<p class="tone-muted" style="margin:0;padding:1rem;">No host has reported its Docker disk yet.</p>
+{% endif %}

plugins/docker/templates/docker/_usage.html

@@ -2,8 +2,11 @@
 
     Docker counts CPU per core (100% = one core busy), so CPU reads as cores,
     toned by its share of the host's CPUs when the host's core count is known.
-    Memory reads in bytes, against a limit only when one is actually set;
+    Memory is what its processes hold, file cache left out (#6030), and
-    mem_pct is then a share of that limit, otherwise of the host's memory. -#}
+    reads in bytes, against a limit only when one is actually set;
+    mem_pct is then a share of that limit, otherwise of the host's memory.
+    Each is toned by usage()'s grade when it has one: within the container's
+    own normal for the hour reads healthy, and says what is usual (#5742). -#}
 
 {% macro cores(n) -%}
 {{ "%.2f"|format(n) if n < 1 else "%.1f"|format(n) }} {{ "core" if n == 1 else "cores" }}
@@ -12,15 +15,15 @@
 {% macro cpu(u) -%}
 {%- if u.cores is none -%}<span class="tone-dim">—</span>
 {%- else -%}
-<span class="tone-{{ status_class(u.cpu_share, 'cpu') }}"
+<span class="tone-{{ u.cpu_grade or status_class(u.cpu_share, 'cpu') }}"
-      title="Docker counts CPU per core: {{ '%.0f'|format(u.cores * 100) }}% of one core{% if u.cpus %}, {{ '%.1f'|format(u.cpu_share) }}% of this host's {{ u.cpus }} core{{ 's' if u.cpus != 1 }}{% endif %}">{{ cores(u.cores) }}</span>
+      title="Docker counts CPU per core: {{ '%.0f'|format(u.cores * 100) }}% of one core{% if u.cpus %}, {{ '%.1f'|format(u.cpu_share) }}% of this host's {{ u.cpus }} core{{ 's' if u.cpus != 1 }}{% endif %}{% if u.cpu_usual %}; {{ u.cpu_usual }}{% endif %}">{{ cores(u.cores) }}</span>
 {%- endif -%}
 {%- endmacro %}
 
 {% macro mem(u) -%}
 {%- if u.mem_bytes is none -%}<span class="tone-dim">—</span>
 {%- else -%}
-<span class="tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of else 'dim' }}"
+<span class="tone-{{ u.mem_grade or (status_class(u.mem_pct, 'mem') if u.mem_of else 'dim') }}"
-      title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% else %}No memory limit set{% endif %}">{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %}</span>
+      title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% if u.mem_usual %}, {{ u.mem_usual }}{% endif %}{% else %}No memory limit set{% endif %}">{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %}</span>
 {%- endif -%}
 {%- endmacro %}

plugins/docker/templates/docker/_usage_history.html

@@ -0,0 +1,23 @@
+{# docker/_usage_history.html — CPU and memory over the selected range, for a
+   service (summed across its replicas, #5302) or one container (#5310): the
+   latest and the peak, over a line that spans its half. The line is drawn
+   against at least cpu_floor / mem_floor, so idling near zero draws flat. #}
+{% from "docker/_usage.html" import cores %}
+{% if have_data %}
+<div class="history-pair">
+  <div>
+    <div class="section-title">CPU</div>
+    <div style="font-variant-numeric:tabular-nums;">{{ cores(cpu_now) }} <span class="tone-dim">now · peak {{ cores(cpu_peak) }}</span></div>
+    <div class="spark" title="Drawn from zero to its peak, or to {{ cores(cpu_floor)|striptags }} when the peak is lower">{{ spark_cpu|safe }}</div>
+  </div>
+  <div>
+    <div class="section-title">Memory</div>
+    <div style="font-variant-numeric:tabular-nums;">{{ mem_now|bytes }} <span class="tone-dim">now · peak {{ mem_peak|bytes }}</span></div>
+    <div class="spark" title="Drawn from zero to its peak, or to {{ mem_floor|bytes }} when the peak is lower">{{ spark_mem|safe }}</div>
+  </div>
+</div>
+{% else %}
+<p class="tone-muted" style="margin:0;font-size:0.85rem;">
+  Not enough samples in the last {{ range_key }} yet. History fills in as the agent reports.
+</p>
+{% endif %}

plugins/docker/templates/docker/container_detail.html

@@ -1,98 +1,104 @@
-{# docker/container_detail.html — full detail page for one container. A swarm
+{# docker/container_detail.html — one container (#5310), shaped like a
-   task is titled by its service and slot, with the raw container name under
+   service's page: a header that says whether it is well and what it is; why
-   it; CPU reads in cores and memory in bytes (see _usage.html). #}
+   it stopped, only while it has; what it moved since it started; its CPU and
+   memory over time (the one home for what it uses now), with its metric
+   anomalies above when it has any; its lifecycle.
+   A swarm task is titled by its service and slot, the raw name under it. #}
 {% extends "base.html" %}
-{% from "_macros.html" import crumbs %}
+{% from "_macros.html" import crumbs, status_badge %}
-{% from "docker/_usage.html" import cpu, mem %}
 {% block title %}{{ label or name }} — Docker — Steward{% endblock %}
-{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (label or name, "")]) }}{% endblock %}
+{% block breadcrumb %}
+{%- if container is not none and container.service_name -%}
+{{ crumbs([("Docker", "/plugins/docker/"), (container.service_name, "/plugins/docker/service/" ~ container.service_name), (label, "")]) }}
+{%- else -%}
+{{ crumbs([("Docker", "/plugins/docker/"), (label or name, "")]) }}
+{%- endif -%}
+{% endblock %}
 {% block content %}
 
 {% if container is none %}
-<div class="card" style="text-align:center;padding:3rem;">
+<div class="card empty-state">
-  <div style="font-weight:500;margin-bottom:0.4rem;">Container not found</div>
+  <strong>Container not found</strong>
-  <div style="color:var(--text-muted);font-size:0.9rem;">
+  <span>No container named <code>{{ name }}</code> is reported for this host now. It may have been
-    No container named <code>{{ name }}</code> is currently reported for this host.
+    removed, or the host agent has not reported recently. <a href="/plugins/docker/">Back to Docker</a></span>
-    It may have been removed, or the host agent hasn't reported recently.
-  </div>
 </div>
 {% else %}
+{% set c = container %}
 
-{# ── Header ──────────────────────────────────────────────────────────────── #}
+{# ── Header: the verdict, then what it is ────────────────────────────────── #}
-<div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
+<div class="page-header">
-  <span class="dot {% if container.status == 'running' %}dot-up{% elif container.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
+  <div style="min-width:0;">
-  <h1 class="page-title" style="margin-bottom:0;">{{ label }}</h1>
+    <h1 class="page-title" style="overflow-wrap:anywhere;">{{ label }}</h1>
-  {% if container.health == 'healthy' %}<span style="font-size:0.72rem;color:var(--green);border:1px solid var(--green);border-radius:3px;padding:0.05rem 0.4rem;">healthy</span>
+    <p class="page-subtitle">
-  {% elif container.health == 'unhealthy' %}<span style="font-size:0.72rem;color:var(--red);border:1px solid var(--red);border-radius:3px;padding:0.05rem 0.4rem;">unhealthy</span>
+      {% if c.status == 'running' %}{{ status_badge('ok', 'Running') }}
-  {% elif container.health == 'starting' %}<span style="font-size:0.72rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0.05rem 0.4rem;">starting</span>{% endif %}
+      {% elif c.status == 'restarting' %}{{ status_badge('crit', 'Restarting') }}
+      {% elif c.status == 'paused' %}{{ status_badge('warn', 'Paused') }}
+      {% elif exit_failed %}{{ status_badge('crit', c.status|capitalize) }}
+      {% else %}{{ status_badge('dim', c.status|capitalize) }}{% endif %}
+      {%- if c.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}
+      {%- elif c.health == 'starting' %} {{ status_badge('pending', 'Starting') }}
+      {%- elif c.health == 'healthy' %} {{ status_badge('ok', 'Healthy') }}{% endif %}
+      {%- if uptime %} · up {{ uptime }}{% endif %}
+      {%- if host %} · on <a href="/hosts/{{ host.id }}">{{ host.name }}</a>{% endif %}
+      {%- if c.service_name %} · task of <a href="/plugins/docker/service/{{ c.service_name }}">{{ c.service_name }}</a>
+      {%- elif c.compose_project %} · compose <span class="chip">{{ c.compose_project }}</span>{% endif %}
+      {%- if c.image %} · <span class="mono" style="overflow-wrap:anywhere;" title="{{ c.image }}">{{ c.image.split("@")[0] }}</span>{% endif %}
+    </p>
+    <p class="page-subtitle tone-dim" style="margin-top:0.2rem;">
+      {%- if label != c.name %}<span class="mono" style="overflow-wrap:anywhere;">{{ c.name }}</span> · {% endif -%}
+      {%- if c.scraped_at %}reported {{ c.scraped_at|ago }}{% endif -%}
+    </p>
   </div>
-{% if label != container.name %}
+  <div class="page-actions"><a class="btn btn-sm btn-ghost" href="/plugins/docker/container/{{ host_id }}/{{ name }}/logs">Logs</a></div>
-<div style="font-size:0.75rem;color:var(--text-dim);font-family:ui-monospace,monospace;margin-bottom:0.25rem;overflow-wrap:anywhere;">{{ container.name }}</div>
-{% endif %}
-<div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
-  {{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %}
-  {% if host %} · on <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %}
-  · <a href="/plugins/docker/container/{{ host_id }}/{{ name }}/logs">View logs</a>
 </div>
 
-{# ── Facts grid ──────────────────────────────────────────────────────────── #}
+{# ── Why it stopped, only while it has ───────────────────────────────────── #}
-{% macro fact(label, value, colour="") %}
+{% if c.status != 'running' and exit_failed %}
-<div class="card" style="margin-bottom:0;">
+<section class="card problem-card" aria-labelledby="ctr-exit">
-  <div class="section-title" style="margin-bottom:0.3rem;">{{ label }}</div>
+  <h2 id="ctr-exit" class="section-title" style="margin-bottom:0.4rem;">Last exit</h2>
-  <div style="font-size:0.95rem;{% if colour %}color:{{ colour }};{% endif %}font-family:ui-monospace,monospace;">{{ value }}</div>
+  <p style="margin:0;">
-</div>
+    Exited <span class="tone-crit">{{ c.exit_code }}</span>{% if c.oom_killed %}: killed for running out of memory{% endif %}
-{% endmacro %}
+    {%- if died_at %} · {{ died_at|ago }}{% endif %}
-
+    · <a href="/plugins/docker/container/{{ host_id }}/{{ name }}/logs">its logs →</a>
-<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
+  </p>
-  {{ fact("CPU", cpu(usage) if container.status == 'running' else "—") }}
+</section>
-  {{ fact("Memory", mem(usage) if container.status == 'running' else "—") }}
-  {#- Swarm never restarts a task's container: it replaces the task with a new
-      one, so restart_count is always 0 there and the service's task failures
-      (Swarm page) are the count that means something. #}
-  {% if not container.service_name %}
-  {{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }}
-  {% endif %}
-  {{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—",
-          "var(--red)" if exit_failed else "") }}
-  {{ fact("Net in since start", net_rx) }}
-  {{ fact("Net out since start", net_tx) }}
-  {{ fact("Disk read since start", blk_read) }}
-  {{ fact("Disk written since start", blk_write) }}
-</div>
-{% if container.service_name %}
-<div style="font-size:0.78rem;color:var(--text-muted);margin:-0.75rem 0 1.5rem;">
-  Swarm replaces a failed task with a new container rather than restarting it; the
-  service's failed tasks are on the <a href="/plugins/docker/swarm">Swarm page</a>.
-</div>
 {% endif %}
 
-{# ── Image / placement ───────────────────────────────────────────────────── #}
+{# ── Its CPU or memory out of its normal for the hour, and past anomalies (#5742) #}
-<div class="card" style="margin-bottom:1.5rem;">
+{% with back=anomaly_back, base_resource=anomaly_resource %}{% include "_anomaly_card.html" %}{% endwith %}
-  <div style="display:grid;grid-template-columns:max-content 1fr;gap:0.4rem 1.25rem;font-size:0.86rem;">
+{# ── Its open tickets ──────────────────────────────────────────────────────── #}
-    <span style="color:var(--text-muted);">Image</span>
+<div hx-get="/tickets/card?subject={{ anomaly_resource|urlencode }}&amp;back={{ anomaly_back|urlencode }}" hx-trigger="load" hx-swap="outerHTML"></div>
-    <span style="font-family:ui-monospace,monospace;word-break:break-all;">{{ container.image or "—" }}</span>
+
-    {% if container.compose_project %}
+{# ── What it is set up with, and what it moved since it started ─────────── #}
-    <span style="color:var(--text-muted);">Compose project</span><span>{{ container.compose_project }}</span>
+<div class="card">
-    {% endif %}
+  <dl class="kv">
-    {% if container.service_name %}
+    {% if not c.service_name %}
-    <span style="color:var(--text-muted);">Swarm service</span><span>{{ container.service_name }}</span>
+    {#- Swarm never restarts a task's container: it replaces the task, so a
-    {% endif %}
+        task's restart count is always 0; its service's Tasks table is the
-    {% if container.node_id %}
+        count that means something. #}
-    <span style="color:var(--text-muted);">Node</span>
+    <dt>Restarts</dt><dd>{% if c.restart_count %}<span class="tone-warn">{{ c.restart_count }}</span>{% else %}0{% endif %}
-    {% if node_name %}<span title="node id {{ container.node_id }}">{{ node_name }}</span>
+      {%- if c.exit_code is not none and c.status == 'running' and c.restart_count %} <span class="tone-dim">· last exit {{ c.exit_code }}</span>{% endif %}</dd>
-    {% else %}<span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>{% endif %}
     {% endif %}
+    <dt>Memory limit</dt><dd>{% if usage.mem_limit %}{{ usage.mem_limit|bytes }}{% else %}<span class="tone-dim">none set</span>{% endif %}</dd>
+    {#- Its memory figure is what its processes hold; the file cache charged to
+        it is shown here only, since the kernel gives it back when the host
+        needs it (#6030). #}
+    <dt>File cache</dt><dd>{% if c.mem_cache_bytes is not none %}{{ c.mem_cache_bytes|bytes }} <span class="tone-dim">files it read or wrote, kept in memory until the host needs it back; not counted in its memory</span>{% else %}<span class="tone-dim">—</span>{% endif %}</dd>
     {% if ports %}
-    <span style="color:var(--text-muted);">Ports</span>
+    <dt>Ports</dt><dd class="mono">{% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %}</dd>
-    <span style="font-family:ui-monospace,monospace;">{% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %}</span>
     {% endif %}
-  </div>
+    {% if c.node_id %}
+    <dt>Swarm node</dt><dd title="node id {{ c.node_id }}">{{ node_name or c.node_id[:12] }}</dd>
+    {% endif %}
+    <dt>Network</dt><dd>{{ net_rx }} in · {{ net_tx }} out <span class="tone-dim">since it started</span></dd>
+    <dt>Disk</dt><dd>{{ blk_read }} read · {{ blk_write }} written <span class="tone-dim">since it started</span></dd>
+  </dl>
 </div>
 
-{# ── Resource history (range-toggled) ────────────────────────────────────── #}
+{# ── Resource history: the one home for what it uses ────────────────────── #}
-<div class="card" style="margin-bottom:1.5rem;">
+<section class="card" aria-labelledby="ctr-history">
   <div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
-    <h3 class="section-title" style="margin-bottom:0;">Resource history</h3>
+    <h2 id="ctr-history" class="section-title" style="margin:0;">Resource history</h2>
     {% include "_time_range.html" %}
   </div>
   <div id="container-history"
@@ -100,31 +106,30 @@
        hx-trigger="load, rangeChange from:body"
        hx-include="#time-range"
        hx-swap="innerHTML">
-    <div style="color:var(--text-muted);font-size:0.85rem;">Loading…</div>
+    <p class="field-hint">Loading…</p>
-  </div>
   </div>
+</section>
 
-{# ── Lifecycle timeline ──────────────────────────────────────────────────── #}
+{# ── Lifecycle, newest first ─────────────────────────────────────────────── #}
-<div class="card">
+<section class="card-flush" aria-labelledby="ctr-lifecycle">
-  <h3 class="section-title" style="margin-bottom:0.75rem;">Lifecycle</h3>
+  <h2 id="ctr-lifecycle" class="table-caption">Lifecycle <span class="tone-muted">newest first</span></h2>
   {% if events %}
-  <div style="display:grid;gap:0.5rem;">
+  <table class="table">
+    <thead><tr><th>Event</th><th>Detail</th><th class="num">When</th></tr></thead>
+    <tbody>
       {% for e in events %}
-    <div style="display:flex;align-items:baseline;gap:0.6rem;font-size:0.85rem;">
+      <tr>
-      <span style="color:{{ e.colour }};width:1rem;text-align:center;flex-shrink:0;">{{ e.glyph }}</span>
+        <td><span class="tone-{{ e.tone }}" aria-hidden="true">{{ e.glyph }}</span> {{ e.event|replace("_", " ") }}</td>
-      <span style="font-weight:500;width:6.5rem;flex-shrink:0;">{{ e.event }}</span>
+        <td class="tone-muted" style="overflow-wrap:anywhere;">{{ e.detail or "" }}</td>
-      <span style="color:var(--text-muted);flex:1;min-width:0;">{{ e.detail or "" }}</span>
+        <td class="num">{{ e.at|ago }}</td>
-      <span style="color:var(--text-dim);font-size:0.78rem;white-space:nowrap;" title="{{ e.at }}">{{ e.at.strftime("%Y-%m-%d %H:%M") }}</span>
+      </tr>
-    </div>
       {% endfor %}
-  </div>
+    </tbody>
+  </table>
   {% else %}
-  <div style="color:var(--text-muted);font-size:0.85rem;">
+  <p class="tone-muted" style="margin:0;padding:1rem;">No lifecycle events recorded yet. Starts, stops and health changes appear here as the agent reports them.</p>
-    No lifecycle events recorded yet. Start/stop/health changes appear here as the
-    agent reports them over time.
-  </div>
   {% endif %}
-</div>
+</section>
 
 {% endif %}
 {% endblock %}

plugins/docker/templates/docker/disk.html

@@ -1,117 +1,124 @@
-{# docker/disk.html — image/disk usage: reclaimable space, per-image sizes #}
+{# docker/disk.html — what Docker keeps on each host's disk (#5304, #5305):
+   the hosts side by side with how much each grew, then each host's parts,
+   its cleanup actions, and its images folded under their repository. #}
 {% extends "base.html" %}
-{% from "_macros.html" import crumbs %}
+{% from "_macros.html" import crumbs, page_header, stat_tile, status_badge %}
 {% block title %}Disk — Docker — Steward{% endblock %}
-{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Image & disk usage", "")]) }}{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Disk", "")]) }}{% endblock %}
 {% block content %}
 
-<h1 class="page-title" style="margin-bottom:0.4rem;">Image &amp; disk usage</h1>
+{% call page_header("Docker disk", "What Docker keeps on each host's disk, from docker system df, and how it has grown.") %}
-<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
+  {% include "_time_range.html" %}
-  Reclaimable = space held by images no container references. Admins can prune
+{% endcall %}
-  per host below; each action runs an audited Ansible playbook on that host.
-</p>
 
 {% if host_groups %}
-{% for g in host_groups %}
+<section class="card-flush" aria-labelledby="disk-hosts">
-<div style="margin-bottom:2rem;">
+  <h2 id="disk-hosts" class="table-caption">By host <span class="tone-muted">most space first</span></h2>
-  <div style="font-weight:500;font-size:0.95rem;margin-bottom:0.75rem;">{{ g.host_name }}</div>
+  <div hx-get="/plugins/docker/disk/hosts" hx-trigger="load, rangeChange from:body"
+       hx-include="#time-range" hx-swap="innerHTML">
+    <p class="field-hint" style="padding:1rem;margin:0;">Loading…</p>
+  </div>
+</section>
 
-  {# ── Summary stats ────────────────────────────────────────────────────── #}
+{% for g in host_groups %}
-  <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1rem;">
+{% set s = g.summary %}
-    <div class="card" style="margin-bottom:0;">
+<section id="host-{{ g.host_id }}" class="disk-host" aria-labelledby="disk-host-{{ g.host_id }}">
-      <div class="section-title" style="margin-bottom:0.3rem;">Reclaimable</div>
+  <h2 id="disk-host-{{ g.host_id }}" class="disk-host-head">
-      <span class="stat-val" style="color:{% if g.summary.images_reclaimable != '0 B' %}var(--orange){% else %}var(--green){% endif %};">{{ g.summary.images_reclaimable }}</span>
+    <a href="/hosts/{{ g.host_id }}">{{ g.host_name }}</a>
-    </div>
+    <span class="tone-muted">{{ g.total|bytes }}</span>
-    <div class="card" style="margin-bottom:0;">
+    {% if g.as_of %}<span class="field-hint">as of {{ g.as_of|ago }}</span>{% endif %}
-      <div class="section-title" style="margin-bottom:0.3rem;">Images</div>
+  </h2>
-      <span class="stat-val">{{ g.summary.images_size }}</span>
+
-      <div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.images_active }}/{{ g.summary.images_total }} in use</div>
+  <div class="stat-strip">
-    </div>
+    {{ stat_tile("Reclaimable", s.images_reclaimable|bytes, state="warn" if s.images_reclaimable else "ok", hint="held by images no container uses") }}
-    <div class="card" style="margin-bottom:0;">
+    {{ stat_tile("Images", s.images_size|bytes, hint=s.images_active ~ " of " ~ s.images_total ~ " in use") }}
-      <div class="section-title" style="margin-bottom:0.3rem;">Stopped</div>
+    {{ stat_tile("Container files", s.containers_size|bytes, hint="written by " ~ s.containers_count ~ " containers on top of their images") }}
-      <span class="stat-val" style="{% if g.stopped %}color:var(--text-muted){% endif %};">{{ g.stopped }}</span>
+    {{ stat_tile("Volumes", s.volumes_size|bytes, hint=s.volumes_count ~ " volumes; never pruned here") }}
-    </div>
+    {{ stat_tile("Build cache", s.build_cache_size|bytes, hint="left by docker build; safe to clear") }}
-    <div class="card" style="margin-bottom:0;">
+    {{ stat_tile("Stopped", g.stopped, state="dim" if not g.stopped else none, hint="containers not running") }}
-      <div class="section-title" style="margin-bottom:0.3rem;">Writable layers</div>
-      <span class="stat-val">{{ g.summary.containers_size }}</span>
-      <div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.containers_count }} containers</div>
-    </div>
-    <div class="card" style="margin-bottom:0;">
-      <div class="section-title" style="margin-bottom:0.3rem;">Volumes</div>
-      <span class="stat-val">{{ g.summary.volumes_size }}</span>
-      <div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.volumes_count }} volumes</div>
-    </div>
-    <div class="card" style="margin-bottom:0;">
-      <div class="section-title" style="margin-bottom:0.3rem;">Build cache</div>
-      <span class="stat-val">{{ g.summary.build_cache_size }}</span>
-    </div>
   </div>
 
   {# ── Cleanup actions (admin only; audited Ansible prune run) ───────────── #}
   {% if session.user_role == 'admin' %}
-  <div style="margin-bottom:1rem;">
+  <div class="disk-actions">
     {% if ansible_available and g.has_target %}
-    <div style="display:flex;flex-wrap:wrap;gap:0.5rem;align-items:center;">
+    <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune"
-      <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
           data-confirm="Remove all STOPPED containers on {{ g.host_name }}? This cannot be undone.">{{ csrf_field() }}
       <input type="hidden" name="target" value="containers">
       <button type="submit" class="btn btn-sm">Prune stopped containers</button>
     </form>
-      <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
+    <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune"
           data-confirm="Remove ALL unused images on {{ g.host_name }} (docker image prune -a)? Any image not used by a container is deleted.">{{ csrf_field() }}
       <input type="hidden" name="target" value="images">
       <button type="submit" class="btn btn-sm">Prune unused images</button>
     </form>
-      <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
+    <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune"
-            data-confirm="Run a full system prune on {{ g.host_name }}? Removes stopped containers, unused networks, dangling images and build cache.">{{ csrf_field() }}
+          data-confirm="Run a full system prune on {{ g.host_name }}? Removes stopped containers, unused networks, untagged images nothing uses, and the build cache.">{{ csrf_field() }}
       <input type="hidden" name="target" value="system">
       <button type="submit" class="btn btn-sm btn-danger">System prune…</button>
     </form>
-    </div>
+    <span class="field-hint">Each runs an audited Ansible playbook on {{ g.host_name }}; reclaimed space shows on the next agent sample.</span>
-    <div style="font-size:0.72rem;color:var(--text-muted);margin-top:0.4rem;">
-      Reclaimed space appears on the next agent sample.
-    </div>
     {% elif not ansible_available %}
-    <div style="font-size:0.74rem;color:var(--text-muted);">Cleanup needs the Ansible runner (currently unavailable).</div>
+    <span class="field-hint">Cleanup needs the Ansible runner (currently unavailable).</span>
     {% else %}
-    <div style="font-size:0.74rem;color:var(--text-muted);">
+    <span class="field-hint">Link an <a href="/hosts/{{ g.host_id }}">Ansible target</a> to this host to enable prune actions.</span>
-      Link an <a href="/hosts/{{ g.host_id }}">Ansible target</a> to this host to enable prune actions.
-    </div>
     {% endif %}
   </div>
   {% endif %}
 
-  {# ── Per-image table ──────────────────────────────────────────────────── #}
+  {# ── Images by repository, versions folded under it (#5305) ─────────── #}
-  <div class="card-flush">
+  <section class="card-flush" aria-label="Images on {{ g.host_name }}">
-    <table class="table">
+    <h3 class="table-caption">Images by repository
-      <thead>
+      <span class="tone-muted">{% if g.images_capped %}the {{ g.images_listed }} largest of {{ s.images_total }} images{% else %}{{ g.images_listed }} images{% endif %}</span></h3>
-        <tr><th>Image</th><th>Size</th><th>Shared</th><th>Containers</th></tr>
+    {% if g.repos %}
-      </thead>
+    <table class="table image-repos">
-      <tbody>
+      <thead><tr>
-        {% for im in g.images %}
+        <th>Repository</th>
+        <th class="num">Versions</th>
+        <th class="num" title="Each version's size less the layers it shares with other images, added up: what removing them all would free, roughly">Own size</th>
+        <th class="num">Used by</th>
+        <th class="num" title="Own size of the versions no container uses">Reclaimable</th>
+      </tr></thead>
+      {% for r in g.repos %}
+      <tbody class="image-repo">
         <tr>
-          <td style="font-size:0.84rem;font-family:ui-monospace,monospace;max-width:340px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
+          <td class="cell-name">
-            {{ im.repo_tag }}
+            <details>
-            {% if im.reclaimable %}<span title="not referenced by any container" style="font-size:0.68rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0 0.3rem;margin-left:0.4rem;">reclaimable</span>{% endif %}
+              <summary title="{{ r.repo or 'untagged images' }}">{% if r.repo %}<span class="mono">{{ r.repo }}</span>{% else %}<span class="tone-muted">untagged</span>{% endif %}</summary>
-          </td>
+              <table class="table image-versions">
-          <td style="font-family:ui-monospace,monospace;font-size:0.86rem;">{{ im.size }}</td>
+                <thead><tr><th>Tag</th><th>ID</th><th class="num">Size</th><th class="num" title="Layers in common with other images: stored once">Shared</th><th class="num">Used by</th></tr></thead>
-          <td style="font-family:ui-monospace,monospace;font-size:0.86rem;color:var(--text-muted);">{{ im.shared }}</td>
+                <tbody>
-          <td style="font-size:0.86rem;color:{% if im.containers %}var(--text){% else %}var(--text-muted){% endif %};">{{ im.containers }}</td>
+                  {% for v in r.versions %}
+                  <tr>
+                    <td class="mono">{{ v.tag or "—" }}</td>
+                    <td class="mono tone-muted">{{ (v.image_id or "")|replace("sha256:", "")|truncate(12, true, "") }}</td>
+                    <td class="num">{{ v.size|bytes }}</td>
+                    <td class="num tone-muted">{{ v.shared|bytes }}</td>
+                    <td class="num">{% if v.containers %}{{ v.containers }} container{{ "s" if v.containers != 1 }}{% else %}<span class="tone-warn">nothing</span>{% endif %}</td>
                   </tr>
-        {% else %}
-        <tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No images reported.</td></tr>
                   {% endfor %}
                 </tbody>
               </table>
-  </div>
+            </details>
-</div>
+          </td>
+          <td class="num">{{ r.versions|length }}{% if r.unused %} <span class="tone-dim">({{ r.unused }} unused)</span>{% endif %}</td>
+          <td class="num">{{ r.own|bytes }}</td>
+          <td class="num">{% if r.containers %}{{ r.containers }} container{{ "s" if r.containers != 1 }}{% else %}<span class="tone-warn">nothing</span>{% endif %}</td>
+          <td class="num">{% if r.reclaimable %}<span class="tone-warn">{{ r.reclaimable|bytes }}</span>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        </tr>
+      </tbody>
+      {% endfor %}
+    </table>
+    {% else %}
+    <p class="tone-muted" style="margin:0;padding:1rem;">No images reported.</p>
+    {% endif %}
+  </section>
+</section>
 {% endfor %}
 {% else %}
-<div class="card" style="text-align:center;padding:3rem;">
+<div class="card empty-state">
-  <div style="color:var(--text-muted);font-size:0.9rem;">
+  No disk usage reported yet. The host agent reports what Docker keeps on disk,
-    No disk usage reported yet. The host agent reports image/disk usage from
+  from <code>docker system df</code>, on hosts running Docker.
-    <code>docker system df</code> on hosts running Docker.
-  </div>
 </div>
 {% endif %}
 {% endblock %}

plugins/docker/templates/docker/host_panel.html

@@ -1,19 +1,13 @@
 {# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub:
-   problems first with what is wrong and since when, then running, then
+   problems first with what is wrong and since when, then the busiest running
-   stopped. Tasks Swarm has replaced are left out (counted in the header). #}
+   containers; the rest of the running and the stopped fold away so the card
+   stays short (#5276). Tasks Swarm has replaced are left out (counted in the
+   header). #}
 {% from "_macros.html" import status_badge, status_dot %}
 {% from "docker/_usage.html" import cpu, mem %}
-<div class="card">
+{% macro container_table(rows) %}
-  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;flex-wrap:wrap;">
-    <h3 class="section-title" style="margin-bottom:0;">Docker</h3>
-    <span style="font-size:0.78rem;color:var(--text-muted);">
-      {% if problems %}<span class="tone-crit">{{ problems }} with problems</span> · {% endif %}
-      {{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %}
-      {% if replaced %}<span title="Old swarm tasks Swarm has replaced with new ones: history, not containers that need anything"> · {{ replaced }} replaced task{{ "s" if replaced != 1 }} hidden</span>{% endif %}
-      <a href="/plugins/docker/" style="margin-left:0.6rem;color:var(--text-muted);">All →</a>
-    </span>
-  </div>
 <table class="table">
+  <colgroup><col><col style="width:7rem"><col style="width:9rem"></colgroup>
   <thead>
     <tr>
       <th>Container</th>
@@ -55,4 +49,28 @@
       {% endfor %}
   </tbody>
 </table>
+{% endmacro %}
+<div class="card">
+  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;flex-wrap:wrap;">
+    <h3 class="section-title" style="margin-bottom:0;">Docker</h3>
+    <span style="font-size:0.78rem;color:var(--text-muted);">
+      {% if problems %}<span class="tone-crit">{{ problems }} with problems</span> · {% endif %}
+      {{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %}
+      {% if replaced %}<span title="Old swarm tasks Swarm has replaced with new ones: history, not containers that need anything"> · {{ replaced }} replaced task{{ "s" if replaced != 1 }} hidden</span>{% endif %}
+      <a href="/plugins/docker/" style="margin-left:0.6rem;color:var(--text-muted);">All →</a>
+    </span>
+  </div>
+  {{ container_table(rows) }}
+  {% if folded %}
+  <details style="margin-top:0.5rem;">
+    <summary style="cursor:pointer;font-size:0.82rem;color:var(--text-muted);">
+      {%- set more_running = folded|selectattr("c.status", "equalto", "running")|list|length -%}
+      {%- set more_stopped = folded|length - more_running -%}
+      {%- if more_running %}+{{ more_running }} more running{% endif -%}
+      {%- if more_running and more_stopped %} · {% endif -%}
+      {%- if more_stopped %}{{ more_stopped }} stopped{% endif -%}
+    </summary>
+    {{ container_table(folded) }}
+  </details>
+  {% endif %}
 </div>

plugins/docker/templates/docker/index.html

@@ -1,5 +1,8 @@
 {% extends "base.html" %}
 {% from "_macros.html" import page_header %}
+{# The Docker page. The filter bar sits outside the polled fragment, so a poll
+   never resets what is being typed; the fragment reads the bar on every poll,
+   and a change re-fetches it and puts the filters in the URL (#5299). #}
 {% block title %}Docker — Steward{% endblock %}
 {% block content %}
 {% call page_header("Docker", "What needs attention first, then every service and container.") %}
@@ -7,17 +10,66 @@
   {% if has_disk %}<a class="btn btn-sm btn-ghost" href="/plugins/docker/disk">Disk</a>{% endif %}
 {% endcall %}
 
+<form id="docker-filters" class="filter-bar" method="get" action="/plugins/docker/" role="search">
+  <input type="search" id="docker-q" name="q" value="{{ f.q }}" class="input"
+         placeholder="Search name, image or stack" aria-label="Search services and containers" autocomplete="off">
+  {% if choices.stacks %}
+  <select name="stack" aria-label="Stack">
+    <option value="">All stacks</option>
+    {% for s in choices.stacks %}<option value="{{ s }}"{% if f.stack == s %} selected{% endif %}>{{ s }}</option>{% endfor %}
+    <option value="{{ no_stack }}"{% if f.stack == no_stack %} selected{% endif %}>No stack</option>
+  </select>
+  {% endif %}
+  {% if choices.hosts|length > 1 %}
+  <select name="host" aria-label="Host or node">
+    <option value="">All hosts</option>
+    {% for h in choices.hosts %}<option value="{{ h }}"{% if f.host == h %} selected{% endif %}>{{ h }}</option>{% endfor %}
+  </select>
+  {% endif %}
+  <select name="state" aria-label="Status">
+    <option value="">Any status</option>
+    {% for key, label in states.items() %}<option value="{{ key }}"{% if f.state == key %} selected{% endif %}>{{ label }}</option>{% endfor %}
+  </select>
+  {% if f.sort != 'status' %}<input type="hidden" name="sort" value="{{ f.sort }}"><input type="hidden" name="dir" value="{{ f.dir }}">{% endif %}
+  <noscript><button type="submit" class="btn btn-sm">Filter</button></noscript>
+</form>
+
 <div id="docker-rows"
      hx-get="/plugins/docker/rows"
-     hx-trigger="load, every {{ poll_interval }}s"
+     hx-include="#docker-filters"
+     hx-trigger="load, every {{ poll_interval }}s, change from:#docker-filters, input changed delay:250ms from:#docker-q, search from:#docker-q"
      hx-swap="innerHTML">
   <div class="card"><p class="field-hint">Loading…</p></div>
 </div>
-{# A link to a section (the dashboard's "failed (24h)" → #failures) lands
+{% if disk %}
-   before the rows exist; scroll to it once, when they first arrive. #}
+{# What Docker keeps on each host's disk, and its growth over a week (#5306). #}
+<p class="docker-disk-line">
+  <a href="/plugins/docker/disk" class="section-title">Disk</a>
+  {% for d in disk %}
+  <span class="docker-disk-host"><a href="/plugins/docker/disk#host-{{ d.host_id }}">{{ d.host_name }}</a>
+    {{ d.total|bytes }}
+    {%- if d.growth and d.growth.change %} <span class="tone-{{ 'warn' if d.growth.pct is not none and d.growth.pct >= 10 else 'muted' }}"
+      title="Since {{ d.growth.since.strftime('%Y-%m-%d %H:%M UTC') }}">{{ "+" if d.growth.change > 0 else "−" }}{{ (d.growth.change|abs)|bytes }} in 7d</span>{% endif -%}
+    {%- if d.reclaimable %} <span class="tone-dim">· {{ d.reclaimable|bytes }} reclaimable</span>{% endif %}</span>
+  {% endfor %}
+</p>
+{% endif %}
 <script>
 (function () {
   var rows = document.getElementById('docker-rows');
+  var form = document.getElementById('docker-filters');
+  // The filters live in the URL too, so a filtered view can be bookmarked and
+  // Back returns to it.
+  function keepInUrl() {
+    var q = new URLSearchParams(new FormData(form));
+    Array.from(q.keys()).forEach(function (k) { if (!q.get(k)) q.delete(k); });
+    var s = q.toString();
+    history.replaceState(null, '', location.pathname + (s ? '?' + s : '') + location.hash);
+  }
+  form.addEventListener('change', keepInUrl);
+  form.addEventListener('input', keepInUrl);
+  // A link to a section (the dashboard's "failed (24h)" → #failures) lands
+  // before the rows exist; scroll to it once, when they first arrive.
   if (!location.hash || !rows) return;
   rows.addEventListener('htmx:afterSwap', function once() {
     rows.removeEventListener('htmx:afterSwap', once);

plugins/docker/templates/docker/rows.html

@@ -1,13 +1,36 @@
-{# docker/rows.html — HTMX fragment for the Docker page: what needs attention
+{# docker/rows.html — HTMX fragment for the Docker page (#5298–#5301):
-   first (service.needs_attention, the ranking the MCP tools use), the counts,
+   a verdict line; what needs
-   then swarm services and each host's containers. #}
+   attention (service.needs_attention, the ranking the MCP tools use); hosts
-{% from "_macros.html" import stat_tile, status_badge, status_dot %}
+   whose Docker is not answering; what failed in 24h, each with its logs; then
+   the swarm services and the standalone containers as two tables that share
+   the filter bar's search, filters and sort (overview.py). #}
+{% from "_macros.html" import status_badge, status_dot, sort_th %}
 {% from "docker/_usage.html" import cpu, mem, cores %}
+{% macro th(key, label, cls=none, title=none) -%}
+{{ sort_th(label, sort_url(key), active=f.sort == key, desc=f.dir == 'desc', cls=cls, title=title) }}
+{%- endmacro %}
+{% macro up(row) -%}
+{%- if row.started_at %}<span title="Started {{ row.started_at.strftime('%Y-%m-%d %H:%M UTC') }}">{{ row.started_at|ago|striptags|replace(" ago", "") }}</span>
+{%- else %}<span class="tone-dim">—</span>{% endif -%}
+{%- endmacro %}
+
+{# ── Verdict ──────────────────────────────────────────────────────────────── #}
+{% set problems = attention|length + failing_services|length %}
+<p class="page-verdict">
+  {% if problems %}<strong class="tone-crit">{{ problems }} need{{ "s" if problems == 1 }} attention</strong>
+  {% else %}<strong class="tone-ok">Nothing needs attention</strong>{% endif %}
+  {%- if services_total %} · <span class="tone-{{ 'ok' if services_healthy == services_total else 'warn' }}">{{ services_healthy }}/{{ services_total }}</span> services healthy{% endif %}
+  {%- if containers_total %} · {{ running }} container{{ "s" if running != 1 }} running{% if stopped %}, {{ stopped }} stopped{% endif %}{% if services_total %} outside services{% endif %}{% endif %}
+  {%- if docker_hosts %} · {{ docker_hosts }} Docker host{{ "s" if docker_hosts != 1 }}{% endif %}
+  {%- if stale %} · <span class="tone-warn" title="No recent report from their agent: what is shown may no longer be true">{{ stale }} stale</span>{% endif %}
+</p>
 
 {# ── Needs attention ──────────────────────────────────────────────────────── #}
 {% if attention or failing_services %}
 <section class="card problem-card" aria-labelledby="docker-problems">
-  <h2 id="docker-problems" class="section-title" style="margin-bottom:0.6rem;">Needs attention</h2>
+  <h2 id="docker-problems" class="section-title section-count">Needs attention <span class="tone-crit">{{ problems }}</span>
+    {%- if unhealthy %}<span class="tone-muted"> · {{ unhealthy }} unhealthy</span>{% endif %}
+    {%- if restarting %}<span class="tone-muted"> · {{ restarting }} restarting</span>{% endif %}</h2>
   <ul class="problem-list">
     {# A swarm service in trouble: Swarm replaces its failed tasks, so the
        service, not any one container, is what is failing. #}
@@ -17,13 +40,14 @@
       {% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
       {% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}
       {% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
-      <a href="/plugins/docker/swarm">{{ s.name }}</a>
+      <a href="/plugins/docker/service/{{ s.name }}">{{ s.name }}</a>
       <span class="problem-detail">
         {{- s.running }} of {{ s.desired }} running
         {%- if s.crash_looping %} · {{ s.failures_1h }} tasks failed in the last hour{% endif %}
         {%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.exit_code %} (exit {{ e.exit_code }}){% endif %}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
         {%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
       </span>
+      <a class="field-hint" href="/plugins/docker/service/{{ s.name }}{% if e and e.task_id %}?task={{ e.task_id }}{% endif %}#logs" style="margin-left:0.4rem;">Logs →</a>
     </li>
     {% endfor %}
     {% for r in attention %}
@@ -38,6 +62,7 @@
         {%- if r.since %} · since {{ r.since|ago }}{% endif %}
         {%- if r.stale %} · {{ status_badge('stale') }} last report {{ c.scraped_at|ago if c.scraped_at else "never" }}{% endif -%}
       </span>
+      <a class="field-hint" href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}/logs" style="margin-left:0.4rem;">Logs →</a>
     </li>
     {% endfor %}
   </ul>
@@ -60,170 +85,144 @@
 </section>
 {% endif %}
 
-{# ── Counts ──────────────────────────────────────────────────────────────
-   Swarm services are counted as services (a task's container comes and goes);
-   the container counts are the containers outside any swarm service. ────── #}
-<div class="stat-strip">
-  {{ stat_tile("Needs attention", attention|length + failing_services|length, state="crit" if attention or failing_services else "inactive") }}
-  {{ stat_tile("Unhealthy", unhealthy, state="crit" if unhealthy else "inactive") }}
-  {{ stat_tile("Restarting", restarting, state="crit" if restarting else "inactive", hint="restarting or crash-looping") }}
-  {{ stat_tile("Failed (24h)", failures|length, state="warn" if failures else "inactive", hint="containers and services; listed below", href="#failures" if failures else none) }}
-  {% if swarm_services %}{{ stat_tile("Swarm services", services_healthy ~ "/" ~ swarm_services|length, state="up" if services_healthy == swarm_services|length else "warn", hint="healthy", href="/plugins/docker/swarm") }}{% endif %}
-  {{ stat_tile("Containers running", running, state="up" if running else "inactive", hint="outside swarm services" if swarm_services else none) }}
-  {{ stat_tile("Containers stopped", stopped, state="inactive") }}
-  {% if stale %}{{ stat_tile("Stale", stale, state="stale", hint="no recent report") }}{% endif %}
-  {{ stat_tile("Docker hosts", docker_hosts) }}
-</div>
-
 {# ── What failed in the last 24h: what the dashboard's "failed (24h)" counts.
    A plain container's death, or a swarm service's failed tasks; a container
-   someone stopped (exit 137/143) is not a failure. ───────────────────────── #}
+   someone stopped (exit 137/143) is not a failure. Each links to the logs of
+   what died: a service's to its failed task's lines (#5300). ───────────── #}
 {% if failures %}
-<section id="failures" class="card" aria-labelledby="docker-failures" style="margin-bottom:1.5rem;">
+<section id="failures" class="card" aria-labelledby="docker-failures">
-  <h2 id="docker-failures" class="section-title" style="margin-bottom:0.6rem;">Failed in the last 24h</h2>
+  <h2 id="docker-failures" class="section-title section-count">Failed in the last 24h <span class="tone-warn">{{ failures|length }}</span></h2>
   <ul class="problem-list">
-    {% for f in failures %}
+    {% for x in failures %}
     <li>
-      {{ status_badge('warn', 'Service' if f.kind == 'service' else 'Container') }}
+      {{ status_badge('warn', 'Service' if x.kind == 'service' else 'Container') }}
-      {% if f.kind == 'service' %}<a href="/plugins/docker/swarm">{{ f.name }}</a>
+      {% if x.kind == 'service' %}<a href="/plugins/docker/service/{{ x.name }}">{{ x.name }}</a>
-      {% else %}<a href="/plugins/docker/container/{{ f.host_id }}/{{ f.name }}">{{ f.name }}</a>{% endif %}
+      {% else %}<a href="/plugins/docker/container/{{ x.host_id }}/{{ x.name }}">{{ x.name }}</a>{% endif %}
       <span class="problem-detail">
-        {{- f.count }} {{ "failure" if f.count == 1 else "failures" }}
+        {{- x.count }} {{ "failure" if x.count == 1 else "failures" }}
-        {%- if f.detail %} · last: {{ f.detail }}{% endif %}
+        {%- if x.detail %} · last: {{ x.detail }}{% endif %}
-        {%- if f.host %} · {{ f.host }}{% endif %}
+        {%- if x.host %} · {{ x.host }}{% endif %}
-        {%- if f.at %} · {{ f.at|ago }}{% endif -%}
+        {%- if x.at %} · {{ x.at|ago }}{% endif -%}
       </span>
+      {% if x.kind == 'service' %}
+      <a class="field-hint" href="/plugins/docker/service/{{ x.name }}{% if x.task_id %}?task={{ x.task_id }}{% endif %}#logs" style="margin-left:0.4rem;">Logs{% if x.task_id %} of that task{% endif %} →</a>
+      {% else %}
+      <a class="field-hint" href="/plugins/docker/container/{{ x.host_id }}/{{ x.name }}/logs" style="margin-left:0.4rem;">Logs →</a>
+      {% endif %}
     </li>
     {% endfor %}
   </ul>
 </section>
 {% endif %}
 
-{# ── Swarm services: collapsed per-service, each current replica labelled with
+{# ── Swarm services: one row each, problems first unless a header picked
-   its host (tasks Swarm replaced are left out). Ghost replicas (tasks on nodes
+   another order; CPU and memory add up the running replicas (#5299). ───── #}
-   with no Steward agent) come from the managers' placement data so the picture
+{% if services %}
-   is cluster-complete. A failing service names Swarm's last task error. ───── #}
+<section class="card-flush" aria-labelledby="docker-services">
-{% if swarm_services %}
+  <h2 id="docker-services" class="table-caption">Swarm services <span class="tone-muted">{{ services|length }}{% if services|length != services_total %} of {{ services_total }}{% endif %}</span>
-<div class="card-flush" style="margin-bottom:1.5rem;">
+    <a href="/plugins/docker/swarm" class="field-hint">By node →</a></h2>
-  <div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
-    <span style="font-weight:500;font-size:0.95rem;">Swarm services</span>
-    <a href="/plugins/docker/swarm" style="font-size:0.78rem;color:var(--text-muted);">Topology →</a>
-  </div>
-  <div style="padding:0.5rem 0.75rem;display:grid;gap:0.85rem;">
-    {% for s in swarm_services %}
-    <div>
-      <div style="display:flex;align-items:center;gap:0.55rem;flex-wrap:wrap;margin-bottom:0.35rem;">
-        <span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
-        <span style="font-weight:500;font-size:0.9rem;">{{ s.name }}</span>
-        <span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
-        <span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-size:0.78rem;font-variant-numeric:tabular-nums;">
-          {{ s.running }}/{{ s.desired }} running{% if s.state != 'healthy' and s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif %}
-        </span>
-      </div>
-      {% with style="padding-left:1.1rem;margin-bottom:0.35rem;" %}{% include "docker/_swarm_failures.html" %}{% endwith %}
-      <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(210px,1fr));gap:0.4rem;padding-left:1.1rem;">
-        {% for r in s.replicas %}
-        {% if r.ghost %}
-        <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:5px;padding:0.3rem 0.55rem;" title="Running on a node with no Steward agent — detail unavailable">
-          <span class="dot dot-warn" style="opacity:0.6;"></span>
-          <span style="font-weight:500;">{{ r.host }}</span>
-          <span style="color:var(--text-dim);margin-left:auto;">{{ r.count }} · no agent</span>
-        </div>
-        {% else %}
-        <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px solid var(--border);border-radius:5px;padding:0.3rem 0.55rem;">
-          <span class="dot {% if r.status == 'running' %}dot-up{% elif r.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-          <a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" style="font-weight:500;color:inherit;text-decoration:none;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" title="{{ r.name }}">{{ r.host }}</a>
-          {% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;" title="Docker counts CPU per core">{{ cores(r.cpu_pct / 100) }}</span>{% endif %}
-        </div>
-        {% endif %}
-        {% endfor %}
-      </div>
-    </div>
-    {% endfor %}
-  </div>
-</div>
-{% endif %}
-
-{# ── Container tables, one per host ───────────────────────────────────────── #}
-{% if host_groups %}
-{% for g in host_groups %}
-<div class="card-flush" style="margin-bottom:1.5rem;">
-  <div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
-    {% if g.host %}
-    <a href="/hosts/{{ g.host.id }}" style="font-weight:500;font-size:0.95rem;color:inherit;text-decoration:none;">{{ g.host.name }}</a>
-    {% else %}
-    <span style="font-weight:500;font-size:0.95rem;">{{ g.host_name }}</span>
-    {% endif %}
-    <span style="font-size:0.78rem;color:var(--text-muted);">
-      {{ g.running }} running{% if g.stopped %} · {{ g.stopped }} stopped{% endif %}
-    </span>
-  </div>
   <table class="table">
     <thead>
       <tr>
-        <th>Container</th>
+        {{ th("name", "Service") }}
-        <th>Image</th>
+        {{ th("status", "Status", title="Problems first: a crash loop, then down, then short of replicas, then failed tasks in 24h") }}
-        <th>Ports</th>
+        {{ th("stack", "Stack", title="The part of the service's name before the first _: the stack it was deployed in") }}
-        <th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
+        <th title="The nodes running its replicas">Nodes</th>
-        <th title="Used, and the limit when one is set">Memory</th>
+        {{ th("cpu", "CPU", "num", "Across its running replicas; Docker counts CPU per core") }}
+        {{ th("mem", "Memory", "num", "Across its running replicas") }}
+        {{ th("up", "Up", "num", "Since its newest replica started") }}
+        <th class="num" title="Tasks that failed in the last 24h; Swarm replaced them">Failed 24h</th>
       </tr>
     </thead>
     <tbody>
-      {% for sub in g.subgroups %}
+      {% for r in services %}
-      {% if g.grouped and sub.label %}
-      <tr><td colspan="5" style="padding:0.4rem 0.75rem 0.2rem;font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;font-weight:500;">{{ sub.label }}</td></tr>
-      {% endif %}
-      {% for item in sub.containers %}
-      {% set c = item.container %}
       <tr>
+        <td class="cell-name"><a href="/plugins/docker/service/{{ r.name }}" title="{{ r.name }}{% if r.image %} · {{ r.image }}{% endif %}">{{ r.name }}</a></td>
         <td>
-          <div style="display:flex;align-items:center;gap:0.5rem;">
+          {{ status_dot('up' if r.state == 'healthy' else ('warn' if r.state == 'degraded' else 'down')) }}
-            {{ status_dot('stale' if item.stale else ('up' if c.status == 'running' else ('pending' if c.status == 'paused' else 'down')), 'Stale' if item.stale else c.status|capitalize) }}
+          <span class="tone-{{ 'ok' if r.state == 'healthy' else ('warn' if r.state == 'degraded' else 'crit') }}" style="font-variant-numeric:tabular-nums;">{{ r.running }}/{{ r.desired }}</span>
-            <div>
+          {%- if r.crash_looping %} {{ status_badge('crit', 'Crash-looping') }}{% elif r.unhealthy %} {{ status_badge('crit', 'Unhealthy') }}{% endif %}
-              <div style="font-weight:500;font-size:0.9rem;">
+          {%- if r.mode == 'global' %} <span class="tone-dim">global</span>{% endif %}
-                <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="color:inherit;text-decoration:none;border-bottom:1px dotted var(--border-mid);">{{ c.name }}</a>
-                {% if c.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}
-                {% elif c.health == 'starting' %} {{ status_badge('pending', 'Starting') }}{% endif %}
-                {% if item.stale %} {{ status_badge('stale', title="Last report " ~ (c.scraped_at.strftime('%Y-%m-%d %H:%M UTC') if c.scraped_at else 'never') ~ "; what is shown may no longer be true") }}{% endif %}
-              </div>
-              <div style="font-size:0.73rem;color:var(--text-muted);">
-                {{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
-                {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
-                  · <span class="tone-{{ 'crit' if item.exit_failed else 'dim' }}" title="{{ 'last exit code' if item.exit_failed else 'stopped by a signal; not a failure' }}">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
-                {% endif %}
-                {% if c.restart_count %} · <span class="tone-warn" title="Restarted {{ c.restart_count }} times">⟳{{ c.restart_count }}</span>{% endif %}
-              </div>
-              {% if c.service_name or c.compose_project %}
-              <div style="font-size:0.68rem;color:var(--text-dim);margin-top:0.1rem;">
-                {{ c.service_name or c.compose_project }}
-              </div>
-              {% endif %}
-            </div>
-          </div>
         </td>
-        <td style="font-size:0.82rem;color:var(--text-muted);max-width:200px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
+        <td>{% if r.stack %}<a class="chip" href="{{ page_url(stack=r.stack) }}" title="Show the {{ r.stack }} stack">{{ r.stack }}</a>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
-          {{ c.image }}
+        <td class="tone-muted">
+          {{- r.nodes|join(", ") or ("" if r.ghost_nodes else "—") -}}
+          {%- if r.ghost_nodes %}{{ ", " if r.nodes }}<span title="Running on a node with no Steward agent: no figures for it">{{ r.ghost_nodes|join(", ") }} (no agent)</span>{% endif -%}
         </td>
-        <td style="font-size:0.78rem;font-family:ui-monospace,monospace;">
+        <td class="num">{% if r.cpu_cores is not none %}{{ cores(r.cpu_cores) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
-          {% for p in item.ports %}
+        <td class="num">{% if r.mem_bytes is not none %}{{ r.mem_bytes|bytes }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
-          <div style="color:var(--text-muted);">{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}</div>
+        <td class="num">{{ up(r) }}</td>
-          {% endfor %}
+        <td class="num">
+          {%- if r.failures_24h %}<a class="tone-warn" href="/plugins/docker/service/{{ r.name }}#tasks"
+               title="{% if r.last_error %}Last: {{ r.last_error.error or r.last_error.message or r.last_error.state }}{% if r.last_error.exit_code %} (exit {{ r.last_error.exit_code }}){% endif %}{% endif %}">{{ r.failures_24h }}</a>
+          {%- else %}<span class="tone-dim">—</span>{% endif -%}
         </td>
-        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;white-space:nowrap;">{{ cpu(item.usage) }}</td>
-        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;white-space:nowrap;">{{ mem(item.usage) }}</td>
       </tr>
       {% endfor %}
-      {% endfor %}
     </tbody>
   </table>
-</div>
+</section>
+{% endif %}
+
+{# ── Containers outside any swarm service, every host in one table (#5301);
+   a compose project is its stack. ─────────────────────────────────────── #}
+{% if containers %}
+<section class="card-flush" aria-labelledby="docker-containers">
+  <h2 id="docker-containers" class="table-caption">{{ "Other containers" if services_total else "Containers" }} <span class="tone-muted">{{ containers|length }}{% if containers|length != containers_total %} of {{ containers_total }}{% endif %}</span></h2>
+  <table class="table">
+    <thead>
+      <tr>
+        {{ th("name", "Container") }}
+        {{ th("status", "Status", title="Problems first: unhealthy or restarting, then died with a failing exit, then stale; then running, then stopped") }}
+        <th>Host</th>
+        {{ th("stack", "Stack", title="Its compose project") }}
+        <th>Image</th>
+        {{ th("cpu", "CPU", "num", "Docker counts CPU per core: 1 core = one core fully busy") }}
+        {{ th("mem", "Memory", "num", "Used, and the limit when one is set") }}
+        {{ th("up", "Up", "num", "Since it started") }}
+      </tr>
+    </thead>
+    <tbody>
+      {% for r in containers %}
+      <tr>
+        <td class="cell-name">
+          <a href="/plugins/docker/container/{{ r.host_id }}/{{ r.container_name }}" title="{{ r.container_name }}">{{ r.name }}</a>
+          {%- if r.ports %}<div class="cell-sub mono">{% for p in r.ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{{ " " if not loop.last }}{% endfor %}</div>{% endif %}
+        </td>
+        <td>
+          {{ status_dot('stale' if r.stale else ('up' if r.running else ('pending' if r.status == 'paused' else 'down'))) }}
+          {%- if r.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}
+          {%- elif r.health == 'starting' %} {{ status_badge('pending', 'Starting') }}
+          {%- else %} <span class="tone-muted">{{ r.status }}</span>{% endif %}
+          {%- if not r.running and r.exit_code is not none and r.exit_code != 0 %}
+          <span class="tone-{{ 'crit' if r.exit_failed else 'dim' }}" title="{{ 'last exit code' if r.exit_failed else 'stopped by a signal; not a failure' }}">exit {{ r.exit_code }}{% if r.oom_killed %} (OOM){% endif %}</span>
+          {%- endif %}
+          {%- if r.restart_count %} <span class="tone-warn" title="Restarted {{ r.restart_count }} times">⟳{{ r.restart_count }}</span>{% endif %}
+          {%- if r.stale %} {{ status_badge('stale', title="Last report " ~ (r.scraped_at.strftime('%Y-%m-%d %H:%M UTC') if r.scraped_at else 'never') ~ "; what is shown may no longer be true") }}{% endif %}
+        </td>
+        <td class="tone-muted">{{ r.host }}</td>
+        <td>{% if r.stack %}<a class="chip" href="{{ page_url(stack=r.stack) }}" title="Show the {{ r.stack }} stack">{{ r.stack }}</a>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="cell-image" title="{{ r.image }}">{{ r.image }}</td>
+        <td class="num">{% if r.running %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="num">{% if r.running %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="num">{{ up(r) }}</td>
+      </tr>
       {% endfor %}
-{% elif not swarm_services %}
+    </tbody>
-<div class="card" style="text-align:center;padding:3rem;">
+  </table>
-  <div style="color:var(--text-muted);font-size:0.9rem;">
+</section>
+{% endif %}
+
+{% if not services and not containers %}
+<div class="card">
+  {% if services_total or containers_total %}
+  <p class="tone-muted" style="margin:0;">Nothing matches these filters. <a href="/plugins/docker/">Show everything</a></p>
+  {% else %}
+  <p class="tone-muted" style="margin:0;">
     No containers reported yet. Containers are collected by the Steward host agent —
     deploy the agent to a host running Docker (Hosts → the host → agent panel) and
-    its containers will appear here under that host.
+    its containers will appear here.
-  </div>
+  </p>
+  {% endif %}
 </div>
 {% endif %}

plugins/docker/templates/docker/service.html

@@ -0,0 +1,160 @@
+{# docker/service.html — one swarm service (#5302, #5310): a header that says
+   whether it is well and what it runs; its last failure only while it is in
+   trouble; its CPU and memory over time; one table of its tasks, the running
+   ones first with what they use; and the logs of every task it has run.
+   A failure's "Logs →" lands on #logs with ?task= picked. #}
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs, status_badge %}
+{% from "docker/_usage.html" import cpu, mem %}
+{% block title %}{{ name }} — Docker — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
+{% block content %}
+
+{% if service is none %}
+<div class="card empty-state">
+  <strong>{{ "No longer running" if task_choices else "Service not found" }}</strong>
+  {% if task_choices %}
+  <span>No swarm service named <code>{{ name }}</code> is reported now; it may have been removed.
+    The logs Steward still keeps from its tasks are below.</span>
+  {% else %}
+  <span>No swarm service named <code>{{ name }}</code> is reported, and Steward holds no logs or
+    events for one. It may have been removed. <a href="/plugins/docker/">Back to Docker</a></span>
+  {% endif %}
+</div>
+{% else %}
+
+{# ── Header: the verdict, then what it is (#5310). Each fact has one home:
+   what it uses is in Resource history, its tasks and failures in Tasks. #}
+{% set troubled = row.crash_looping or row.state != 'healthy' %}
+{% set tone = 'ok' if row.state == 'healthy' else ('warn' if row.state == 'degraded' else 'crit') %}
+{% set image = (svc.image if svc and svc.image else row.image) %}
+{% set e = row.last_error %}
+<div class="page-header">
+  <div style="min-width:0;">
+    <h1 class="page-title" style="overflow-wrap:anywhere;">{{ name }}</h1>
+    <p class="page-subtitle">
+      {% if row.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
+      {% elif row.state == 'down' %}{{ status_badge('crit', 'Down') }}
+      {% elif row.state == 'degraded' %}{{ status_badge('warn', 'Degraded') }}
+      {% else %}{{ status_badge('ok', 'Healthy') }}{% endif %}
+      <span class="tone-{{ tone }}">{{ row.running }} of {{ row.desired }} running</span>
+      {%- if troubled and row.trouble_since %} · <span class="tone-{{ tone }}">in trouble since {{ row.trouble_since|ago }}</span>{% endif %}
+      · {{ row.mode }}
+      {%- if stack %} · stack <a class="chip" href="/plugins/docker/?stack={{ stack }}">{{ stack }}</a>{% endif %}
+      {%- if image %} · <span class="mono" style="overflow-wrap:anywhere;" title="{{ image }}">{{ image.split("@")[0] }}</span>{% endif %}
+    </p>
+    {% if (not troubled and e) or svc %}
+    <p class="page-subtitle tone-dim" style="margin-top:0.2rem;">
+      {%- if not troubled and e %}Last failed {% if e.age_s is not none %}{{ e.age_s|duration }} ago{% endif %}
+      {%- if e.exit_code %} (exit {{ e.exit_code }}){% endif %}
+      {%- if e.task_id %} · <a href="?task={{ e.task_id }}#logs">its logs</a>{% endif %} · {% endif -%}
+      {%- if svc %}reported {{ svc.updated_at|ago }} by the manager's agent{% endif -%}
+    </p>
+    {% endif %}
+  </div>
+  <div class="page-actions"><a class="btn btn-sm btn-ghost" href="#logs">Logs</a></div>
+</div>
+
+{# ── What is wrong, only while it is (#5310) ──────────────────────────────── #}
+{% if troubled and e %}
+<section class="card problem-card" aria-labelledby="svc-last-error">
+  <h2 id="svc-last-error" class="section-title" style="margin-bottom:0.4rem;">Last failed task</h2>
+  <p style="margin:0;">
+    {{ e.error or e.message or e.state }}{% if e.exit_code %} <span class="tone-crit">(exit {{ e.exit_code }})</span>{% endif %}
+    {%- if e.node %} · on {{ e.node }}{% endif %}{% if e.age_s is not none %} · {{ e.age_s|duration }} ago{% endif %}
+    {%- if e.task_id %} · <a href="?task={{ e.task_id }}#logs">its logs →</a>{% endif %}
+  </p>
+</section>
+{% endif %}
+
+{# ── Resource history, summed across replicas ────────────────────────────── #}
+<section class="card" aria-labelledby="svc-history">
+  <div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
+    <h2 id="svc-history" class="section-title" style="margin:0;">Resource history <span class="tone-dim">across running replicas</span></h2>
+    {% include "_time_range.html" %}
+  </div>
+  <div hx-get="/plugins/docker/service/{{ name }}/history"
+       hx-trigger="load, rangeChange from:body"
+       hx-include="#time-range"
+       hx-swap="innerHTML">
+    <p class="field-hint">Loading…</p>
+  </div>
+</section>
+
+{# ── Tasks: the running ones with what they use, then Swarm's record ─────── #}
+<section id="tasks" class="card-flush" aria-labelledby="svc-tasks">
+  <h2 id="svc-tasks" class="table-caption">Tasks
+    <span class="tone-muted">running first, then Swarm's most recent</span>
+    {%- if row.failures_24h %} <span class="tone-warn">· {{ row.failures_24h }} failed in 24h</span>{% endif %}</h2>
+  {% if tasks %}
+  <table class="table">
+    <thead><tr>
+      <th title="The replica slot this task filled">Slot</th><th>Task</th><th>Node</th><th>State</th>
+      <th class="num" title="Docker counts CPU per core">CPU</th><th class="num">Memory</th>
+      <th class="num">Exit</th><th>Why</th><th class="num" title="When it started, for a running task; else when it reached its state">When</th>
+    </tr></thead>
+    <tbody>
+      {% for t in tasks %}
+      <tr>
+        <td>{{ t.slot or "—" }}</td>
+        <td class="mono" title="{{ t.container or t.task_id }}">
+          {%- if t.task_id %}<a href="?task={{ t.task_id }}#logs" title="Its logs">{{ t.task_id[:10] }}</a>
+          {%- if t.container and t.host_id %} <a class="tone-dim" href="/plugins/docker/container/{{ t.host_id }}/{{ t.container }}" title="The container's own page">↗</a>{% endif %}
+          {%- elif t.container and t.host_id %}<a href="/plugins/docker/container/{{ t.host_id }}/{{ t.container }}">{{ t.container|truncate(24, true, "…") }}</a>
+          {%- else %}<span class="tone-dim">—</span>{% endif -%}
+        </td>
+        <td>{% if t.host_id %}<a href="/hosts/{{ t.host_id }}">{{ t.node }}</a>{% else %}{{ t.node }}{% endif %}</td>
+        <td>
+          {%- if t.failed %}{{ status_badge('crit', t.state|capitalize) }}
+          {%- elif t.current %}{{ status_badge('ok', 'Running') }}
+          {%- else %}<span class="tone-muted">{{ t.state }}</span>{% endif %}
+          {%- if t.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}{% elif t.health == 'starting' %} {{ status_badge('pending', 'Starting') }}{% endif %}
+          {%- if t.desired and t.desired != t.state and not t.current %} <span class="tone-dim" title="What Swarm wants of it">→ {{ t.desired }}</span>{% endif -%}
+        </td>
+        <td class="num">{% if t.usage %}{{ cpu(t.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="num">{% if t.usage %}{{ mem(t.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="num">{% if t.exit_code %}<span class="tone-{{ 'crit' if t.failed else 'dim' }}">{{ t.exit_code }}</span>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td style="overflow-wrap:anywhere;">{{ t.why or "" }}</td>
+        <td class="num">{% if t.at %}{{ t.at|ago }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p class="tone-muted" style="margin:0;padding:1rem;">No task is running, and Swarm's record of its tasks has not been reported yet (the manager's agent reports it from version 1.8.0).</p>
+  {% endif %}
+</section>
+
+{% endif %}
+
+{# ── Logs across every task, newest first ─────────────────────────────────
+   Shown for a removed service too, while its lines are still kept. #}
+{% if service is not none or task_choices %}
+<section id="logs" class="card-flush" aria-labelledby="svc-logs">
+  <h2 id="svc-logs" class="table-caption">Logs <span class="tone-muted">every task, newest first</span></h2>
+  <form id="svc-log-controls" class="filter-bar" onsubmit="return false;" style="padding:0.75rem 1rem 0;">
+    <input type="search" name="q" class="input" placeholder="Filter lines…" autocomplete="off" aria-label="Filter log lines">
+    {% if task_choices|length > 1 %}
+    <select name="task" aria-label="Task">
+      <option value="">All tasks</option>
+      {% for tid, label in task_choices %}<option value="{{ tid }}"{% if tid == focus_task %} selected{% endif %}>{{ label }}</option>{% endfor %}
+    </select>
+    {% elif focus_task %}<input type="hidden" name="task" value="{{ focus_task }}">{% endif %}
+    <select name="stream" aria-label="Stream">
+      <option value="all">stdout and stderr</option>
+      <option value="stdout">stdout</option>
+      <option value="stderr">stderr</option>
+    </select>
+  </form>
+  <div id="svc-log-lines"
+       hx-get="/plugins/docker/service/{{ name }}/logs/lines"
+       hx-trigger="load, every 5s, change from:#svc-log-controls, keyup changed delay:400ms from:#svc-log-controls"
+       hx-include="#svc-log-controls"
+       hx-swap="innerHTML"
+       role="log" aria-live="polite" tabindex="0"
+       style="max-height:60vh;overflow:auto;padding:0.5rem 1rem 0.75rem;font-family:var(--fs-font-mono);font-size:0.8rem;line-height:1.5;">
+    <p class="field-hint">Loading…</p>
+  </div>
+</section>
+{% endif %}
+{% endblock %}

plugins/docker/templates/docker/swarm.html

@@ -1,95 +1,103 @@
-{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement.
+{# docker/swarm.html — the Swarm laid out by node (#5303). A verdict, the
-   Services in trouble come first (crash loop, down, short), with since-times;
+   services short of replicas, then one card per node: a node that is down or
-   nodes that are down or not taking work come first. #}
+   not taking work first, then managers. Each card says what its containers use
+   against what the machine has, and lists the services placed on it. #}
 {% extends "base.html" %}
-{% from "_macros.html" import crumbs %}
+{% from "_macros.html" import crumbs, status_badge, status_dot %}
+{% from "docker/_usage.html" import cores as fmt_cores %}
 {% block title %}Swarm — Docker — Steward{% endblock %}
 {% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Swarm", "")]) }}{% endblock %}
 {% block content %}
 
-<h1 class="page-title" style="margin-bottom:1.5rem;">Swarm</h1>
+<h1 class="page-title">Swarm</h1>
 
 {% if swarms %}
 {% for g in swarms %}
-<div style="margin-bottom:2rem;">
+{% set down_nodes = g.nodes|rejectattr("ok")|list %}
-  <div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.75rem;flex-wrap:wrap;">
+<section class="swarm-group" aria-label="Swarm reported by {{ g.managers|join(', ') }}">
-    <span style="font-weight:500;font-size:0.95rem;">Swarm</span>
+  <p class="page-verdict">
-    <span style="font-size:0.78rem;color:var(--text-muted);">
+    {% if down_nodes or g.troubled %}<strong class="tone-crit">{{ down_nodes|length + g.troubled|length }} need{{ "s" if (down_nodes|length + g.troubled|length) == 1 }} attention</strong>
-      reported by {{ g.managers | length }} manager{{ 's' if g.managers | length != 1 }} · {{ g.managers | join(", ") }}
+    {% else %}<strong class="tone-ok">Every node is up and taking work</strong>{% endif %}
-    </span>
+    · {{ g.nodes|length }} node{{ "s" if g.nodes|length != 1 }}
-  </div>
+    · <span class="tone-{{ 'ok' if g.services_healthy == g.services_total else 'warn' }}">{{ g.services_healthy }}/{{ g.services_total }}</span> services healthy
+    <span class="tone-dim">· reported by {{ g.managers|join(", ") }}</span>
+  </p>
 
-  {# ── Services ─────────────────────────────────────────────────────────── #}
+  {% if down_nodes or g.troubled %}
-  <div class="card-flush" style="margin-bottom:1rem;">
+  <section class="card problem-card" aria-label="Needs attention">
-    <table class="table">
+    <ul class="problem-list">
-      <thead>
+      {% for n in down_nodes %}
-        <tr>
+      <li>
-          <th>Service</th><th>Mode</th><th>Replicas</th><th>Image</th><th>Placement</th><th>Task failures</th>
+        {{ status_badge('crit', 'Node ' ~ n.status) if n.status != 'ready' else status_badge('warn', n.availability|capitalize) }}
-        </tr>
+        <span>{{ n.hostname }}</span>
-      </thead>
+        <span class="problem-detail">{{ n.role }} · {{ n.task_count }} task{{ "s" if n.task_count != 1 }} placed on it</span>
-      <tbody>
+      </li>
-        {% for s in g.services %}
-        <tr>
-          <td style="font-weight:500;font-size:0.9rem;">{{ s.name }}</td>
-          <td style="font-size:0.82rem;color:var(--text-muted);">{{ s.mode }}</td>
-          <td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
-            <span style="color:{% if s.healthy %}var(--green){% elif s.down %}var(--red){% elif s.degraded %}var(--orange){% else %}var(--text-muted){% endif %};">
-              {{ s.running }}/{{ s.desired }}
-            </span>
-            {% if s.state != 'healthy' and s.trouble_since %}<div style="font-family:inherit;font-size:0.72rem;color:var(--text-dim);" title="{{ s.trouble_since }}">since {{ s.trouble_since|ago }}</div>{% endif %}
-          </td>
-          <td style="font-size:0.8rem;color:var(--text-muted);max-width:220px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.image or "—" }}</td>
-          <td style="font-size:0.78rem;color:var(--text-muted);">
-            {% for p in s.placement %}{{ p.node }} ({{ p.running }}){% if not loop.last %}, {% endif %}{% else %}—{% endfor %}
-          </td>
-          <td style="max-width:360px;">
-            {% include "docker/_swarm_failures.html" %}
-            {%- if not s.tasks_known %}<span style="font-size:0.78rem;color:var(--text-dim);" title="The reporting manager's agent predates task reporting (1.8.0); update it to see task failures.">—</span>
-            {%- elif not (s.crash_looping or s.failures_24h or (s.last_error and s.state != 'healthy')) %}<span style="font-size:0.78rem;color:var(--text-dim);">none in 24h</span>{% endif %}
-          </td>
-        </tr>
-        {% else %}
-        <tr><td colspan="6" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No services in this swarm.</td></tr>
       {% endfor %}
-      </tbody>
+      {% for s in g.troubled %}
-    </table>
+      <li>
-  </div>
+        {% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}{% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}{% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
+        <a href="/plugins/docker/service/{{ s.name }}">{{ s.name }}</a>
+        {% set e = s.last_error %}
+        <span class="problem-detail">
+          {{- s.running }} of {{ s.desired }} running
+          {%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
+          {%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
+        </span>
+      </li>
+      {% endfor %}
+    </ul>
+  </section>
+  {% endif %}
 
-  {# ── Nodes ────────────────────────────────────────────────────────────── #}
+  <div class="node-grid">
-  <div class="card-flush">
-    <table class="table">
-      <thead>
-        <tr><th>Node</th><th>Role</th><th>Availability</th><th>Status</th></tr>
-      </thead>
-      <tbody>
     {% for n in g.nodes %}
-        <tr>
+    <section class="card node-card{% if not n.ok %} problem-card{% endif %}" aria-labelledby="node-{{ n.node_id }}">
-          <td style="font-weight:500;font-size:0.9rem;">
+      <h2 id="node-{{ n.node_id }}" class="node-card-head">
-            {{ n.hostname or n.node_id }}
+        {{ status_dot('up' if n.status == 'ready' else 'down') }}
-            {% if n.leader %}<span title="cluster leader" style="font-size:0.68rem;color:var(--accent);border:1px solid var(--accent);border-radius:3px;padding:0 0.3rem;margin-left:0.3rem;">leader</span>{% endif %}
+        {% if n.host_id %}<a href="/hosts/{{ n.host_id }}">{{ n.hostname }}</a>{% else %}{{ n.hostname }}{% endif %}
-          </td>
+        <span class="chip">{{ "leader" if n.leader else n.role }}</span>
-          <td style="font-size:0.82rem;color:var(--text-muted);">{{ n.role }}</td>
+        {% if n.availability != 'active' %}{{ status_badge('warn', n.availability|capitalize, title="Swarm places no new tasks here") }}{% endif %}
-          <td style="font-size:0.82rem;color:{% if n.availability == 'active' %}var(--text){% else %}var(--orange){% endif %};">{{ n.availability }}</td>
+      </h2>
-          <td style="font-size:0.82rem;">
+      <p class="node-card-figures">
-            <span class="dot {% if n.status == 'ready' %}dot-up{% else %}dot-down{% endif %}"></span>
+        {% if n.cores_used is not none %}
-            <span style="color:{% if n.status == 'ready' %}var(--green){% else %}var(--red){% endif %};">{{ n.status }}</span>
+        Containers use <strong>{{ fmt_cores(n.cores_used) }}</strong>{% if n.cores %} of {{ n.cores }}{% endif %}
-          </td>
+        · <strong>{{ n.mem_used|bytes }}</strong>{% if n.mem %} of {{ n.mem|bytes }}{% endif %}
-        </tr>
+        {% elif n.agent %}
+        <span class="tone-muted">No swarm task reports figures here yet.</span>
         {% else %}
-        <tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No nodes reported.</td></tr>
+        <span class="tone-muted" title="Install the Steward agent on this node to see what its tasks use">No Steward agent: what runs here is Swarm's placement, without figures.</span>
+        {% endif %}
+      </p>
+      {% if n.tasks %}
+      <table class="table node-tasks">
+        <thead><tr><th>Service</th><th class="num">Tasks</th><th class="num">CPU</th><th class="num">Memory</th></tr></thead>
+        <tbody>
+          {% for t in n.tasks %}
+          <tr>
+            <td class="cell-name">
+              {%- if t.state != 'healthy' %}{{ status_dot('down' if t.state == 'down' else 'warn') }} {% endif -%}
+              <a href="/plugins/docker/service/{{ t.name }}" title="{{ t.name }}">{{ t.name }}</a>
+            </td>
+            <td class="num">{{ t.tasks }}</td>
+            <td class="num">{% if t.cores is not none %}{{ fmt_cores(t.cores) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+            <td class="num">{% if t.mem_bytes is not none %}{{ t.mem_bytes|bytes }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+          </tr>
           {% endfor %}
         </tbody>
       </table>
+      {% else %}
+      <p class="tone-muted" style="margin:0;">No tasks placed here.</p>
+      {% endif %}
+    </section>
+    {% endfor %}
   </div>
-</div>
+</section>
 {% endfor %}
 {% else %}
-<div class="card" style="text-align:center;padding:3rem;">
+<div class="card empty-state">
-  <div style="color:var(--text-muted);font-size:0.9rem;">
   No Swarm topology reported. A Steward host agent running on a Swarm
-    <strong>manager</strong> reports services, nodes, and task placement here —
+  <strong>manager</strong> reports services, nodes and task placement here;
-    workers and non-swarm hosts contribute only their local containers.
+  workers and hosts outside a swarm contribute only their own containers.
-  </div>
 </div>
 {% endif %}
 {% endblock %}

plugins/docker/templates/docker/widget.html

@@ -1,80 +1,81 @@
-{# docker/widget.html — dashboard widget: what Docker is running and what has
+{# docker/widget.html — the dashboard Docker widget (#5252), in the widget-shape
-   failed. Swarm services first, troubled ones at the top with since-times; then
+   order: a verdict line; the services and containers in trouble; what failed
-   the containers outside any service, by host, problems first. The failed count
+   in the last 24h, by name; then everything healthy, one line each, packed
-   links to the Docker page's list of exactly what it counts. #}
+   into as many columns as the widget is wide. Resource use is the Busiest
-{% from "docker/_usage.html" import cpu %}
+   Containers widget's, so it isn't repeated here. #}
+{% from "_macros.html" import status_dot %}
 {% if total_count == 0 %}
-<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No containers reported yet.</p>
+<div class="w-verdict" style="color:var(--text-muted);">No containers reported yet.</div>
 {% else %}
-
+{% set wrong = bad_services|length + bad_containers|length %}
-<div style="display:flex;flex-wrap:wrap;gap:0.4rem 1.25rem;margin-bottom:0.65rem;">
+<div class="w-verdict {{ 'tone-crit' if wrong else ('tone-warn' if failures else 'tone-ok') }}">
-  {% if swarm_services %}
+  {%- if services_total %}{{ services_healthy }}/{{ services_total }} services healthy · {% endif -%}
-  <div title="Swarm services with every replica running">
+  {{ running_count }} other container{{ 's' if running_count != 1 }} running ·
-    <span class="tone-{{ 'ok' if services_healthy == swarm_services|length else 'warn' }}" style="font-size:1.4rem;font-weight:500;line-height:1;">{{ services_healthy }}/{{ swarm_services|length }}</span>
+  {% if failures %}<a href="/plugins/docker/#failures" style="color:inherit;">{{ failures|length }} failed in 24h</a>{% else %}nothing failed in 24h{% endif -%}
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">services healthy</span>
-  </div>
-  {% endif %}
-  <div title="Containers running outside any swarm service">
-    <span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--green);">{{ running_count }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">containers running</span>
-  </div>
-  <a href="/plugins/docker/#failures" style="color:inherit;text-decoration:none;"
-     title="Containers that died or were OOM-killed in the last 24h, and swarm services with failed tasks. A container someone stopped is not counted. Click for the list.">
-    <span class="tone-{{ 'crit' if failures else 'dim' }}" style="font-size:1.4rem;font-weight:500;line-height:1;">{{ failures|length }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;border-bottom:1px dotted var(--border-mid);">failed in 24h</span>
-  </a>
 </div>
 
-{# Swarm services — troubled first, each with its replicas' host chips
+{% if wrong %}
-   (dashed = a node with no Steward agent, count-only from placement). #}
+<div class="w-section">Needs a look</div>
-{% if swarm_services %}
+{% for s in bad_services %}
-<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">Swarm services</div>
+<div class="w-item">
-<div style="display:grid;gap:0.4rem;margin-bottom:0.4rem;">
+  {{ status_dot('degraded' if s.state == 'degraded' else 'down') }}
-  {% for s in swarm_services %}
+  <span class="w-label" title="{{ s.name }}">{{ s.name }}
-  <div>
+    <span class="w-sub {{ 'tone-crit' if s.state != 'degraded' else 'tone-warn' }}">
-    <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
+      {%- if s.crash_looping %}crash-looping: {{ s.failures_1h }} failed in the last hour{% else %}{{ s.running }} of {{ s.desired }} running{% endif %}
-      <span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
+      {%- if s.trouble_since %}, since {{ s.trouble_since|ago }}{% endif %}
-      <span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
+      {%- if s.last_error and (s.last_error.error or s.last_error.message) %} · {{ s.last_error.error or s.last_error.message }}{% endif -%}
-      {% if s.crash_looping %}<span class="tone-crit" style="font-size:0.7rem;font-weight:500;flex-shrink:0;" title="{{ s.failures_1h }} tasks failed in the last hour{% if s.last_error %}: {{ s.last_error.error or s.last_error.message or s.last_error.state }}{% endif %}">crash-looping</span>{% endif %}
+    </span>
-      {% if s.state != 'healthy' and s.trouble_since %}<span class="tone-dim" style="font-size:0.7rem;flex-shrink:0;">since {{ s.trouble_since|ago }}</span>{% endif %}
+  </span>
-      <span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
+  <span class="w-meta" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
 </div>
-    <div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">
-      {% for r in s.replicas %}
-      {% if r.ghost %}
-      <span title="no Steward agent on this node — count from swarm placement" style="font-size:0.68rem;color:var(--text-dim);background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }} ×{{ r.count }}</span>
-      {% else %}
-      <span style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }}</span>
-      {% endif %}
 {% endfor %}
+{% for c in bad_containers %}
+<div class="w-item">
+  {{ status_dot('warn' if c.status == 'restarting' else 'down') }}
+  <span class="w-label" title="{{ c.name }}">
+    {%- if session.user_id %}<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>{% else %}{{ c.name }}{% endif %}
+    <span class="w-sub tone-crit">
+      {%- if c.health == 'unhealthy' %}unhealthy{% elif c.status == 'restarting' %}restarting{% else %}exited {{ c.exit_code }}{% if c.oom_killed %}, out of memory{% endif %}{% endif %}
+      {%- set since = (bad_since or {}).get((c.host_id, c.name)) %}{% if since %} since {{ since|ago }}{% endif %} · {{ host_name.get(c.host_id, c.host_id) }}
+    </span>
+  </span>
 </div>
+{% endfor %}
+{% endif %}
+
+{% if failures %}
+<div class="w-section">Failed in the last 24h</div>
+{% for f in failures[:5] %}
+<div class="w-item">
+  {{ status_dot('failed') }}
+  <span class="w-label" title="{{ f.name }}">{{ f.name }}
+    <span class="w-sub">{% if f.count > 1 %}{{ f.count }} times, last {% endif %}{{ f.at|ago }}{% if f.detail %} · {{ f.detail }}{% endif %} · {{ f.host }}</span>
+  </span>
+</div>
+{% endfor %}
+{% if failures|length > 5 %}<div class="w-more">+{{ failures|length - 5 }} more · <a href="/plugins/docker/#failures">the full list</a></div>{% endif %}
+{% endif %}
+
+{% if healthy %}
+<div class="w-section">Healthy</div>
+<div class="w-grid">
+  {% for h in healthy %}
+  <div class="w-item">
+    {% if h.kind == 'service' %}{% set s = h.s %}
+    {{ status_dot('up') }}
+    <span class="w-label" title="{{ s.name }} — on {{ s.replicas|map(attribute='host')|unique|join(', ') }}">{{ s.name }}
+      <span style="color:var(--text-dim);"> · {{ s.replicas|map(attribute='host')|unique|join(', ') }}</span></span>
+    <span class="w-meta" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
+    {% else %}{% set c = h.c %}
+    {{ status_dot('up' if h.kind == 'container' else 'unknown') }}
+    <span class="w-label" title="{{ c.name }}">
+      {%- if session.user_id %}<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>{% else %}{{ c.name }}{% endif %}
+      <span style="color:var(--text-dim);"> · {{ host_name.get(c.host_id, c.host_id) }}</span></span>
+    {% if h.kind == 'stopped' %}<span class="w-meta">stopped</span>{% endif %}
+    {% endif %}
   </div>
   {% endfor %}
 </div>
+{% if healthy_more %}<div class="w-more">+{{ healthy_more }} more healthy · <a href="/plugins/docker/">all of Docker</a></div>{% endif %}
 {% endif %}
-
-{% if host_groups and swarm_services %}
-<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.6rem 0 0.2rem;">Other containers</div>
-{% endif %}
-{% for g in host_groups %}
-{% if multi_host %}
-<div style="font-size:0.7rem;color:var(--text-dim);margin:0.4rem 0 0.15rem;">{{ g.host_name }}</div>
-{% endif %}
-<div style="display:grid;gap:2px;">
-  {% for item in g.containers %}
-  {% set c = item.c %}
-  <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
-    <span class="dot {% if c.status == 'running' and c.health != 'unhealthy' %}dot-up{% elif c.status in ('paused', 'restarting') %}dot-warn{% else %}dot-down{% endif %}"></span>
-    <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;">
-      {{ c.name }}{% if c.health == 'unhealthy' %} <span class="tone-crit" style="font-size:0.7rem;">unhealthy</span>{% endif %}{% if c.status == 'restarting' %} <span class="tone-warn" style="font-size:0.7rem;">restarting</span>{% endif %}
-    </a>
-    {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
-    <span title="{{ 'last exit code' if item.exit_failed else 'stopped by a signal; not a failure' }}" class="tone-{{ 'crit' if item.exit_failed else 'dim' }}" style="font-size:0.7rem;flex-shrink:0;">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
-    {% endif %}
-    {% if c.status == 'running' %}<span style="font-size:0.72rem;font-family:ui-monospace,monospace;flex-shrink:0;">{{ cpu(item.usage) }}</span>{% endif %}
-  </div>
-  {% endfor %}
-</div>
-{% endfor %}
-
 {% endif %}

plugins/docker/templates/docker/widget_resources.html

@@ -1,42 +1,51 @@
-{# docker/widget_resources.html — dashboard widget: the containers using the
+{# docker/widget_resources.html — the dashboard Busiest Containers widget
-   most CPU (averaged over the last few minutes, so the order holds still),
+   (#5253, #5214): a verdict naming the busiest, then the top containers by CPU
-   each with its CPU in cores and memory in bytes. The CPU bar is a share of
+   (averaged over the last few minutes, so the order holds still). Each figure
-   the host's CPUs; the memory bar a share of the container's limit, or of
+   carries its unit, beside bars of the last few hours. A row's bars are scaled
-   the host's memory when no limit is set. #}
+   to its own peak, but never below a floor (1 core, 1 GiB by default), so a
-{% from "docker/_usage.html" import cpu, mem %}
+   container idling at 0.05 cores reads flat and a spike means real load. #}
+{% macro cores(v) -%}{{ "%.2f"|format(v) if v < 1 else "%.1f"|format(v) }}{%- endmacro %}
+{% macro bars(b, peak_text, roof_text) -%}
+<span class="w-bars w-wide-only" aria-hidden="true"
+      title="{% if b.peak is none %}No samples in the last {{ hours }}h{% else %}Peak {{ peak_text }} in the last {{ hours }}h; bars reach the top at {{ roof_text }}{% endif %}">
+  {%- for h in b.bars %}<i{% if h is none %} class="gap"{% else %} style="height:{{ [h, 4]|max }}%"{% endif %}></i>{% endfor -%}
+</span>
+{%- endmacro %}
 {% if not rows %}
-<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No running containers.</div>
+<div class="w-verdict" style="color:var(--text-muted);">No running containers.</div>
 {% else %}
-{% macro bar(label, pct, tone, text, title) %}
+{% set top = rows[0] %}
-<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;" title="{{ title }}">
+<div class="w-verdict">
-  <span style="font-size:0.62rem;color:var(--text-dim);width:1.9rem;flex-shrink:0;letter-spacing:0.03em;">{{ label }}</span>
+  Busiest: <strong>{{ top.label }}</strong> on {{ top.host_name }},
-  <div style="flex:1;height:6px;background:var(--bg-elevated);border-radius:3px;overflow:hidden;">
+  {% if top.usage.cores is not none %}{{ cores(top.usage.cores) }}
-    {% if pct is not none %}
+  {%- if top.usage.cpus %} of {{ top.usage.cpus }} cores{% else %} cores{% endif %}{% else %}no CPU reading yet{% endif %}.
-    <div style="height:100%;border-radius:3px;width:{{ [[pct, 0]|max, 100]|min }}%;
-                background:{% if tone == 'crit' %}var(--red){% elif tone == 'warn' %}var(--orange){% else %}var(--accent){% endif %};
-                transition:width 0.4s;"></div>
-    {% endif %}
 </div>
-  <span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;min-width:4.5rem;text-align:right;flex-shrink:0;white-space:nowrap;">{{ text }}</span>
+<table class="w-table">
-</div>
+  <colgroup><col><col class="w-wide-only" style="width:7.5rem"><col class="w-barcol"><col class="w-barcol"></colgroup>
-{% endmacro %}
+  <thead><tr><th>Container</th><th class="w-wide-only w-text">Host</th>
-<div style="font-size:0.7rem;color:var(--text-dim);margin-bottom:0.45rem;">Busiest by CPU, {{ minutes }}-minute average</div>
+    <th title="Cores, averaged over the last {{ minutes }} minutes; bars: the last {{ hours }}h">CPU<span class="w-wide-only"> · last {{ hours }}h</span></th>
-<div style="display:grid;gap:0.6rem;">
+    <th title="In use now; bars: the last {{ hours }}h">Memory<span class="w-wide-only"> · last {{ hours }}h</span></th></tr></thead>
+  <tbody>
   {% for r in rows %}
   {% set u = r.usage %}
-  <div>
+  <tr>
-    <div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;" title="{{ r.c.name }}">
+    <td class="w-name" title="{{ r.c.name }}">
-      <a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}" style="color:inherit;text-decoration:none;">{{ r.label }}</a>{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
+      {%- if session.user_id %}<a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}">{{ r.label }}</a>{% else %}{{ r.label }}{% endif -%}
-    </div>
+    </td>
-    {% if u.cores is not none %}
+    <td class="w-wide-only w-text" title="{{ r.host_name }}">{{ r.host_name }}</td>
-    {{ bar("CPU", u.cpu_share, status_class(u.cpu_share, 'cpu'), cpu(u),
+    <td class="tone-{{ (u.cpu_grade or status_class(u.cpu_share, 'cpu')) if u.cpus and u.cores is not none else 'dim' }}"
-           ("%.1f%% of the host's %d core%s" % (u.cpu_share, u.cpus, "" if u.cpus == 1 else "s")) if u.cpus else "Host core count unknown") }}
+        {% if u.cpus and u.cores is not none %}title="{{ '%.1f'|format(u.cpu_share) }}% of {{ r.host_name }}'s {{ u.cpus }} cores{% if u.cpu_usual %}; {{ u.cpu_usual }}{% endif %}"{% endif %}>
-    {% endif %}
+      {{- bars(r.cpu_bars, (cores(r.cpu_bars.peak) ~ " cores") if r.cpu_bars.peak is not none else "", cores(r.cpu_bars.roof) ~ " cores") -}}
-    {% if u.mem_bytes is not none %}
+      <span class="w-fig">{% if u.cores is none %}—{% else %}{{ cores(u.cores) }} cores{% endif %}</span>
-    {{ bar("MEM", u.mem_pct if u.mem_of else none, status_class(u.mem_pct, 'mem') if u.mem_of else 'unknown', mem(u),
+    </td>
-           ("%.1f%% of %s" % (u.mem_pct, u.mem_of)) if u.mem_of and u.mem_pct is not none else "") }}
+    <td class="tone-{{ (u.mem_grade or status_class(u.mem_pct, 'mem')) if u.mem_of and u.mem_bytes is not none else 'dim' }}"
-    {% endif %}
+        {% if u.mem_pct is not none and u.mem_of and u.mem_bytes is not none %}title="{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% if u.mem_usual %}, {{ u.mem_usual }}{% endif %}"{% endif %}>
-  </div>
+      {{- bars(r.mem_bars, r.mem_bars.peak|bytes if r.mem_bars.peak is not none else "", r.mem_bars.roof|bytes) -}}
+      <span class="w-fig">{% if u.mem_bytes is none %}—{% else %}{{ u.mem_bytes|bytes }}{% endif %}</span>
+    </td>
+  </tr>
   {% endfor %}
-</div>
+  </tbody>
+</table>
+<div class="w-note w-wide-only">Bars: {{ (hours * 60 // 24) }}-minute slices, oldest left; each row up to its own peak, at least {{ cores(cpu_floor) }} core{{ 's' if cpu_floor != 1 }} and {{ mem_floor|bytes }}.</div>
 {% endif %}

plugins/docker/timeline.py

@@ -0,0 +1,160 @@
+# plugins/docker/timeline.py
+"""What Docker puts on the timeline, judged from each snapshot as it is
+ingested: a deploy (a container or swarm service running a new image, or a
+container recreated), and a crash-loop as one entry for the whole episode.
+A single restart or death is recorded but hidden. As on the Docker pages, a
+crash loop is Settings → Thresholds' number of deaths within CRASH_WINDOW:
+the one that reaches it opens a crash-loop entry, which each later one
+extends until the container has been quiet for CRASH_WINDOW.
+
+A container belongs to its swarm service when it has one, so a service's
+tasks, replaced on every deploy and every crash, read as one subject.
+"""
+from __future__ import annotations
+
+import logging
+from datetime import timedelta
+
+from sqlalchemy import func, select
+
+from steward.models.timeline import TimelineEntry
+from steward.timeline import catalog
+from steward.timeline.record import record
+
+logger = logging.getLogger(__name__)
+
+TYPES = (
+    ("docker.deployed", "Deployed", "change", "service", True,
+     "A container or swarm service started on a new image, or a container was recreated."),
+    ("docker.crash_loop", "Crash-looping", "observed", "service", True,
+     "A container kept restarting or dying: one entry for the whole episode."),
+    ("docker.restarted", "Container restarted or died", "observed", "service", False,
+     "A single restart, death or out-of-memory kill."),
+)
+
+CRASH_WINDOW = timedelta(hours=1)
+_TROUBLE = ("restart", "die", "oom")
+
+
+def register() -> None:
+    for t in TYPES:
+        catalog.register_type(*t)
+
+
+def unit(c: dict) -> str:
+    """What a container is part of: its swarm service, else itself."""
+    return c.get("service_name") or c.get("name")
+
+
+def short_image(image: str | None) -> str:
+    """An image as a person reads it: the name and tag, and a few characters
+    of the digest when there is one."""
+    if not image:
+        return "an unknown image"
+    name, _, digest = image.partition("@")
+    name = name.rsplit("/", 1)[-1]
+    if digest:
+        return f"{name} ({digest.split(':', 1)[-1][:12]})"
+    return name
+
+
+def deploys(old_rows: list, latest: list[dict]) -> list[dict]:
+    """Each unit whose containers now run an image they didn't, or a
+    container recreated under the same name: {unit, name, image, was,
+    recreated, swarm}. A swarm task landing on a host that ran none of its
+    service before is the scheduler moving it, not a deploy."""
+    images: dict[str, set] = {}
+    ids = {}
+    for r in old_rows:
+        images.setdefault(r.service_name or r.name, set()).add(r.image)
+        ids[r.name] = (r.container_id, r.image)
+    out, seen = [], set()
+    for c in latest:
+        if (c.get("status") or "").lower() != "running" or not c.get("name"):
+            continue
+        key, image = unit(c), c.get("image") or ""
+        if key in seen:
+            continue
+        before = images.get(key)
+        old_id, old_image = ids.get(c["name"], (None, None))
+        if before and image not in before:
+            was = old_image if old_image and old_image != image else sorted(before)[0]
+            out.append({"unit": key, "name": c["name"], "image": image, "was": was,
+                        "recreated": False, "swarm": bool(c.get("service_name"))})
+        elif old_id and c.get("container_id") and old_id != c["container_id"] and not c.get("service_name"):
+            out.append({"unit": key, "name": c["name"], "image": image, "was": old_image,
+                        "recreated": True, "swarm": False})
+        elif before is None and not c.get("service_name"):
+            out.append({"unit": key, "name": c["name"], "image": image, "was": None,
+                        "recreated": False, "swarm": False})
+        else:
+            continue
+        seen.add(key)
+    return out
+
+
+def _deploy_summary(d: dict, host_name: str) -> str:
+    what = f"{d['unit']} on {host_name}"
+    if d["recreated"]:
+        return f"{what}: recreated on {short_image(d['image'])}"
+    if d["was"] is None:
+        return f"{what}: started on {short_image(d['image'])}"
+    new, old = short_image(d["image"]), short_image(d["was"])
+    if d["image"].split("@")[0] == (d["was"] or "").split("@")[0]:
+        return f"{what}: new build of {new}"
+    return f"{what}: {old} → {new}"
+
+
+async def _trouble(session, host, key: str, name: str, at, said: str, deaths: int) -> None:
+    """One restart, death or OOM kill of `key`: extends its crash-loop, opens
+    one when it is the `deaths`-th within CRASH_WINDOW, else a hidden entry."""
+    subject = f"{host.name}/{key}"
+    loop = (await session.execute(select(TimelineEntry).where(
+        TimelineEntry.type == "docker.crash_loop", TimelineEntry.subject == subject,
+        TimelineEntry.until >= at - CRASH_WINDOW).order_by(TimelineEntry.at.desc()).limit(1))
+    ).scalar_one_or_none()
+    if loop is not None:
+        count = (loop.detail or {}).get("count", 0) + 1
+        loop.until = max(loop.until, at)
+        loop.detail = {**(loop.detail or {}), "count": count, "last": said}
+        loop.summary = f"{key} on {host.name} crash-looping: {count} restarts or deaths"
+        return
+    recent = (await session.execute(select(func.count(), func.min(TimelineEntry.at)).where(
+        TimelineEntry.type == "docker.restarted", TimelineEntry.subject == subject,
+        TimelineEntry.at >= at - CRASH_WINDOW))).one()
+    url = f"/plugins/docker/container/{host.id}/{name}"
+    if recent[0] + 1 >= deaths:
+        count = recent[0] + 1
+        record(session, type="docker.crash_loop", at=recent[1] or at, until=at,
+               summary=f"{key} on {host.name} crash-looping: {count} restarts or deaths",
+               hosts=[host.name], subject=subject, actor="Docker",
+               detail={"count": count, "last": said, "url": url})
+        return
+    record(session, type="docker.restarted", at=at, summary=f"{key} on {host.name}: {said}",
+           hosts=[host.name], subject=subject, actor="Docker", detail={"url": url})
+
+
+_SAID = {"restart": "restarted", "die": "died", "oom": "killed, out of memory"}
+
+
+async def from_snapshot(session, host, old_rows: list, latest: list[dict], events: list, at,
+                        deaths: int) -> None:
+    """The entries one snapshot makes: its deploys, and its restarts and
+    deaths (`events`: (name, event, detail, when)), in a savepoint so a
+    failure here never loses the snapshot. `deaths` makes a crash loop."""
+    try:
+        async with session.begin_nested():
+            for d in deploys(old_rows, latest):
+                url = "/plugins/docker/swarm" if d["swarm"] else f"/plugins/docker/container/{host.id}/{d['name']}"
+                record(session, type="docker.deployed", at=at, summary=_deploy_summary(d, host.name),
+                       hosts=[host.name], subject=f"{host.name}/{d['unit']}", actor="Docker",
+                       detail={"image": d["image"], "was": d["was"], "url": url})
+            units = {c.get("name"): unit(c) for c in latest if c.get("name")}
+            units.update({r.name: r.service_name or r.name for r in old_rows if r.name not in units})
+            for name, event, detail, when in events:
+                if event in _TROUBLE:
+                    said = _SAID[event] + (f" ({detail})" if detail else "")
+                    await _trouble(session, host, units.get(name, name), name, when or at, said, deaths)
+                    await session.flush()
+    except Exception:
+        logger.exception("Docker on %s: timeline entries could not be written", host.name)

plugins/homeassistant/__init__.py

@@ -0,0 +1,46 @@
+# plugins/homeassistant/__init__.py
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from quart import Quart
+
+_app: "Quart | None" = None
+
+
+def setup(app: "Quart") -> None:
+    global _app
+    _app = app
+    from .models import HomeAssistantConnection, HomeAssistantSchedule, HomeAssistantUpdate  # noqa: F401 — registers the tables
+    from .timeline import register
+    register()
+
+
+def get_scheduled_tasks() -> list:
+    from .scheduler import make_tick_task
+    return [make_tick_task(_app)]
+
+
+def get_blueprint():
+    from .routes import homeassistant_bp
+    return homeassistant_bp
+
+
+def get_nav() -> list[dict]:
+    return [{"label": "Home Assistant", "href": "/plugins/homeassistant/"}]
+
+
+def get_alert_metrics() -> dict[str, list[str]]:
+    """What the poll records, all under the resource "homeassistant"."""
+    return {"homeassistant": ["is_up", "backup_age_hours", "updates_waiting", "problems_open", "disk_free_pct",
+                              "update_failed"]}
+
+
+def register_mcp_tools(register) -> None:
+    from .mcp_tools import register_tools
+    register_tools(register)
+
+
+async def mcp_overview(db) -> list[dict]:
+    from .mcp_tools import mcp_overview as findings
+    return await findings(db)

plugins/homeassistant/client.py

@@ -0,0 +1,292 @@
+# plugins/homeassistant/client.py
+"""Talking to Home Assistant, with what its own sign-in gave Steward: a refresh
+token kept in the connection, and access tokens made from it that last half an
+hour. Repairs and the Supervisor's reports are only offered on its websocket. Home Assistant bans an address after a few refused requests, so nothing is
+sent without a token and a refusal stops the asking. Every failure comes back as
+one sentence a page can show."""
+from __future__ import annotations
+
+import asyncio
+import json
+import ssl
+import time
+from contextlib import asynccontextmanager
+from urllib.parse import urlencode
+
+import httpx
+from websockets.asyncio.client import connect as ws_connect
+from websockets.exceptions import WebSocketException
+
+from steward.core.crypto import decrypt_secret
+
+TIMEOUT_SECONDS = 15.0
+# Home Assistant answers an action when it has finished, minutes later for a
+# backup or an update, and carries on if the request ends. So its answer is only
+# waited for this long, and the outcome is read from its entities.
+ACTION_READ_SECONDS = 10.0
+MAX_BYTES = 16 * 1024 * 1024
+# An access token this close to running out isn't used for another request.
+EXPIRY_MARGIN_SECONDS = 60
+# One read of the system over the websocket, however its answers dribble in.
+SYSTEM_READ_SECONDS = 30
+
+_access: dict[str, tuple[str, float]] = {}      # refresh token -> access token, when to stop using it
+
+
+class Unreachable(Exception):
+    """Home Assistant didn't answer usefully; it may be restarting."""
+
+
+class Refused(Exception):
+    """Home Assistant refused Steward's sign-in, or has banned this address."""
+
+
+class Rejected(Exception):
+    """Home Assistant understood an action and wouldn't do it."""
+
+
+def _refusal(status: int) -> Refused | None:
+    if status == 401:
+        _access.clear()
+        return Refused("Home Assistant refused Steward's sign-in. Sign in again, as a user who may do this.")
+    if status == 403:
+        return Refused("Home Assistant answered 403: it may have banned Steward's address "
+                       "after refused requests. Look in its ip_bans.yaml.")
+    return None
+
+
+async def _request(method: str, url: str, path: str, verify_tls: bool, transport, *, token: str | None = None,
+                   read_timeout: float = TIMEOUT_SECONDS, **kwargs) -> tuple[int, bytes]:
+    headers = {"Authorization": f"Bearer {token}"} if token else {}
+    timeout = httpx.Timeout(TIMEOUT_SECONDS, read=read_timeout)
+    try:
+        # A slow drip of bytes can keep each read inside its own timeout for ever.
+        async with asyncio.timeout(TIMEOUT_SECONDS + read_timeout):
+            async with httpx.AsyncClient(timeout=timeout, verify=verify_tls, transport=transport,
+                                         follow_redirects=False) as client:
+                async with client.stream(method, url.rstrip("/") + path, headers=headers, **kwargs) as response:
+                    body = bytearray()
+                    async for chunk in response.aiter_bytes():
+                        body.extend(chunk)
+                        if len(body) > MAX_BYTES:
+                            raise Unreachable("Home Assistant's answer was too large to read.")
+                    return response.status_code, bytes(body)
+    except TimeoutError:
+        raise httpx.ReadTimeout("no complete answer in time") from None
+
+
+def _unreachable(exc: httpx.HTTPError) -> Unreachable:
+    if isinstance(exc, httpx.TimeoutException):
+        return Unreachable(f"Home Assistant didn't answer within {TIMEOUT_SECONDS:.0f} s.")
+    return Unreachable(f"Couldn't reach Home Assistant: {exc}")
+
+
+# ── Signing in ───────────────────────────────────────────────────────────────
+
+def sign_in_url(url: str, client_id: str, redirect_uri: str, state: str) -> str:
+    """Home Assistant's own sign-in page, which sends the browser back to
+    `redirect_uri`. It accepts `client_id` as it is because the two share an
+    address, so it needn't be able to reach Steward."""
+    return f"{url.rstrip('/')}/auth/authorize?" + urlencode(
+        {"response_type": "code", "client_id": client_id, "redirect_uri": redirect_uri, "state": state})
+
+
+async def _tokens(url: str, form: dict, verify_tls: bool, transport) -> dict:
+    try:
+        status, body = await _request("POST", url, "/auth/token", verify_tls, transport, data=form)
+    except httpx.HTTPError as exc:
+        raise _unreachable(exc) from None
+    if status in (400, 401, 403):
+        raise Refused("Home Assistant no longer accepts Steward's sign-in. Sign in again.")
+    if status != 200:
+        raise Unreachable(f"Home Assistant answered {status}.")
+    try:
+        found = json.loads(body)
+        found["access_token"], int(found["expires_in"])
+    except (ValueError, KeyError, TypeError):
+        raise Unreachable("Home Assistant's answer wasn't a sign-in.") from None
+    return found
+
+
+def _remember(refresh_token: str, found: dict) -> str:
+    until = time.monotonic() + int(found["expires_in"]) - EXPIRY_MARGIN_SECONDS
+    _access[refresh_token] = (found["access_token"], until)
+    return found["access_token"]
+
+
+async def redeem_code(url: str, client_id: str, code: str, verify_tls: bool, transport=None) -> tuple[str, str]:
+    """(refresh token, access token) for the code the sign-in page sent back."""
+    found = await _tokens(url, {"grant_type": "authorization_code", "client_id": client_id, "code": code},
+                          verify_tls, transport)
+    refresh_token = found.get("refresh_token")
+    if not refresh_token:
+        raise Unreachable("Home Assistant's answer wasn't a sign-in.")
+    return refresh_token, _remember(refresh_token, found)
+
+
+async def access_token(connection, transport=None) -> str:
+    refresh_token = decrypt_secret(connection.refresh_token, context="Home Assistant sign-in")
+    token, until = _access.get(refresh_token, (None, 0.0))
+    if token and time.monotonic() < until:
+        return token
+    found = await _tokens(connection.url, {"grant_type": "refresh_token", "client_id": connection.client_id,
+                                           "refresh_token": refresh_token}, connection.verify_tls, transport)
+    return _remember(refresh_token, found)
+
+
+async def is_admin(url: str, token: str, verify_tls: bool, transport=None) -> bool:
+    """Whether the signed-in user is an administrator, which its API only says by
+    refusing someone who isn't. So this is asked once, right after signing in, and
+    a refusal here counts once toward a ban."""
+    try:
+        status, _ = await _request("POST", url, "/api/template", verify_tls, transport, token=token,
+                                   json={"template": "1"})
+    except httpx.HTTPError as exc:
+        raise _unreachable(exc) from None
+    if status == 401:
+        return False
+    if status != 200:
+        raise (_refusal(status) or Unreachable(f"Home Assistant answered {status}."))
+    return True
+
+
+async def revoke(connection, transport=None) -> None:
+    """End this sign-in on Home Assistant. It answers 200 whether or not it knew it."""
+    refresh_token = decrypt_secret(connection.refresh_token, context="Home Assistant sign-in")
+    _access.pop(refresh_token, None)
+    try:
+        status, _ = await _request("POST", connection.url, "/auth/revoke", connection.verify_tls, transport,
+                                   data={"token": refresh_token})
+    except httpx.HTTPError as exc:
+        raise _unreachable(exc) from None
+    if status != 200:
+        raise Unreachable(f"Home Assistant answered {status}.")
+
+
+# ── Reading and acting ───────────────────────────────────────────────────────
+
+async def _get(url: str, path: str, token: str, verify_tls: bool, transport) -> list | dict | None:
+    """The parsed JSON of a GET, or None for a 404."""
+    try:
+        status, body = await _request("GET", url, path, verify_tls, transport, token=token)
+    except httpx.HTTPError as exc:
+        raise _unreachable(exc) from None
+    if refusal := _refusal(status):
+        raise refusal
+    if status == 404:
+        return None
+    if status != 200:
+        raise Unreachable(f"Home Assistant answered {status}.")
+    try:
+        return json.loads(body)
+    except ValueError:
+        raise Unreachable("Home Assistant's answer wasn't JSON.") from None
+
+
+async def read_states(url: str, token: str, verify_tls: bool, transport=None) -> list[dict]:
+    states = await _get(url, "/api/states", token, verify_tls, transport)
+    if not isinstance(states, list):
+        raise Unreachable("Home Assistant's answer wasn't a list of entities.")
+    return states
+
+
+async def read_state(url: str, entity_id: str, token: str, verify_tls: bool, transport=None) -> dict | None:
+    return await _get(url, f"/api/states/{entity_id}", token, verify_tls, transport)
+
+
+async def call_action(url: str, domain: str, action: str, data: dict, token: str, verify_tls: bool,
+                      transport=None) -> None:
+    """Ask Home Assistant to do something. Returning means it got the request,
+    not that it has finished: a request cut off once sent has been taken up, and
+    an update that restarts Home Assistant cuts its own request off."""
+    try:
+        status, body = await _request("POST", url, f"/api/services/{domain}/{action}", verify_tls, transport,
+                                      token=token, read_timeout=ACTION_READ_SECONDS, json=data)
+    except (httpx.ReadTimeout, httpx.ReadError, httpx.RemoteProtocolError):
+        return
+    except httpx.HTTPError as exc:
+        raise _unreachable(exc) from None
+    if refusal := _refusal(status):
+        raise refusal
+    if status != 200:
+        try:
+            said = json.loads(body).get("message")
+        except (ValueError, AttributeError):
+            said = None
+        raise Rejected(said or f"Home Assistant answered {status}.")
+
+
+# ── The system, over the websocket ───────────────────────────────────────────
+
+class _Socket:
+    """Home Assistant's websocket after sign-in: one question at a time, each with
+    an id, and None for one it won't answer (no Supervisor, or not an administrator)."""
+
+    def __init__(self, ws):
+        self.ws, self.asked = ws, 0
+
+    async def sign_in(self, token: str) -> None:
+        await self.ws.recv()
+        await self.ws.send(json.dumps({"type": "auth", "access_token": token}))
+        if json.loads(await self.ws.recv()).get("type") != "auth_ok":
+            raise Refused("Home Assistant refused Steward's sign-in. Sign in again, as a user who may do this.")
+
+    async def ask(self, **question):
+        self.asked += 1
+        await self.ws.send(json.dumps({"id": self.asked, **question}))
+        while True:
+            answer = json.loads(await self.ws.recv())
+            if answer.get("id") == self.asked:
+                return answer.get("result") if answer.get("success") else None
+
+    async def supervisor(self, endpoint: str):
+        return await self.ask(type="supervisor/api", endpoint=endpoint, method="get")
+
+
+@asynccontextmanager
+async def _signed_in(url: str, token: str, verify_tls: bool, connect):
+    """A signed-in socket. `connect` stands in for the websocket in tests."""
+    target = "ws" + url.rstrip("/").removeprefix("http") + "/api/websocket"
+    options: dict = {"max_size": MAX_BYTES}
+    if target.startswith("wss:") and not verify_tls:
+        options["ssl"] = ssl.create_default_context()
+        options["ssl"].check_hostname, options["ssl"].verify_mode = False, ssl.CERT_NONE
+    try:
+        async with asyncio.timeout(SYSTEM_READ_SECONDS):
+            async with (connect or ws_connect)(target, **options) as ws:
+                socket = _Socket(ws)
+                await socket.sign_in(token)
+                yield socket
+    except TimeoutError:
+        raise Unreachable(f"Home Assistant's system report didn't arrive within {SYSTEM_READ_SECONDS} s.") from None
+    except (OSError, ValueError, KeyError, WebSocketException) as exc:
+        raise Unreachable(f"Couldn't read Home Assistant's system report: {exc}") from None
+
+
+async def read_system(url: str, token: str, verify_tls: bool, connect=None) -> dict:
+    """What service.read_health needs, as Home Assistant answered it."""
+    async with _signed_in(url, token, verify_tls, connect) as socket:
+        found = {"config": await socket.ask(type="get_config"),
+                 "issues": (await socket.ask(type="repairs/list_issues") or {}).get("issues"),
+                 "resolution": await socket.supervisor("/resolution/info"),
+                 "host": await socket.supervisor("/host/info"), "titles": {}, "boots": {}}
+        domains = sorted({i["domain"] for i in found["issues"] or []})
+        if domains:
+            titles = await socket.ask(type="frontend/get_translations", language="en",
+                                      category="issues", integration=domains)
+            found["titles"] = (titles or {}).get("resources") or {}
+        found["addons"] = (await socket.supervisor("/addons") or {}).get("addons")
+        # Whether one should be running is only on its own page, so a running one isn't asked.
+        for addon in found["addons"] or []:
+            if addon.get("state") != "started":
+                info = await socket.supervisor(f"/addons/{addon['slug']}/info") or {}
+                found["boots"][addon["slug"]] = info.get("boot")
+        return found
+
+
+async def read_os(url: str, token: str, verify_tls: bool, connect=None) -> dict | None:
+    """The Supervisor's report on the OS: `version`, and `version_pending` while a
+    written update waits for its reboot. None if it isn't answering yet, as just
+    after a boot."""
+    async with _signed_in(url, token, verify_tls, connect) as socket:
+        return await socket.supervisor("/os/info")

plugins/homeassistant/mcp_tools.py

@@ -0,0 +1,168 @@
+"""The Home Assistant plugin's MCP tools and its get_overview findings.
+
+get_home_assistant reads what the poll stored. update_home_assistant starts an
+update as the page's Install does. The sign-in Steward holds is never returned.
+"""
+from __future__ import annotations
+
+from datetime import timezone
+
+from sqlalchemy import select
+
+from steward.core import weekly
+from steward.core.formatting import zone
+from steward.mcp._context import current_caller, mcp_app
+from steward.mcp.tools._audit import audit_mcp, mcp_username
+from steward.mcp.tools._shape import age_s, db_session, iso, now_utc, redact_url
+from steward.mcp.tools.overview import finding
+from steward.models.users import UserRole
+
+from . import schedule
+from .models import HomeAssistantConnection, HomeAssistantSchedule, HomeAssistantUpdate
+from .routes import connection_says
+from .service import STALE_BACKUP_HOURS, backup_age_hours, backup_verdict, installable, parse_moment, problems_open
+from .updates import CannotStart, start_update
+
+RECENT_UPDATES = 10
+
+
+def _schedule_view(plan, now) -> dict | None:
+    if plan is None:
+        return None
+    return {"enabled": plan.enabled, "when": weekly.describe(plan),
+            "next_slot_at": iso(weekly.next_slot(plan, now)) if plan.enabled else None,
+            "last_slot_at": iso(weekly.fired_slot(plan)), "stopped": plan.stopped}
+
+
+def _run_view(r: HomeAssistantUpdate) -> dict:
+    return {"update_id": r.id, "entity_id": r.entity_id, "title": r.title, "kind": r.kind,
+            "from_version": r.from_version, "to_version": r.to_version, "state": r.state, "message": r.message,
+            "started_by": r.started_by, "started_at": iso(r.started_at), "backup_at": iso(r.backup_at),
+            "finished_at": iso(r.finished_at)}
+
+
+async def get_home_assistant() -> dict:
+    """Home Assistant as of the last poll: whether it answers, which updates are
+    waiting, how old its last backup is, and the updates Steward was asked to install.
+
+    `installable` is what update_home_assistant takes (Core, OS, add-ons);
+    `others_waiting` is firmware and custom integrations, which Steward leaves
+    to Home Assistant. `can_install` is whether the Home Assistant user who signed
+    Steward in is an administrator, which installing needs. `system.problems` is
+    what's wrong with the system itself: open repairs, Supervisor health, low disk,
+    recovery mode, add-ons that should be running. `schedule` is the weekly slot
+    for installing what's waiting, null until one is set.
+    """
+    now = now_utc()
+    async with db_session() as db:
+        c = (await db.execute(select(HomeAssistantConnection))).scalars().first()
+        runs = list((await db.execute(select(HomeAssistantUpdate).order_by(
+            HomeAssistantUpdate.started_at.desc()).limit(RECENT_UPDATES))).scalars())
+        plan = (await db.execute(select(HomeAssistantSchedule))).scalars().first()
+    if c is None:
+        return {"connected": False, "says": "No connection yet: an admin signs in on the Home Assistant page."}
+    reach, says = connection_says(c, now)
+    backup = c.backup or {}
+    age = backup_age_hours(backup, now)
+    return {"connected": True, "url": redact_url(c.url), "reach": reach, "says": says,
+            "polled_at": iso(c.last_polled_at), "ok_age_s": age_s(c.last_ok_at, now), "can_install": c.can_install,
+            "installable": [{k: u[k] for k in ("entity_id", "title", "kind", "installed", "latest", "in_progress")}
+                            for u in installable(c)],
+            "others_waiting": [u["title"] for u in c.updates or [] if u["waiting"] and not u["kind"]],
+            "backup": {"verdict": backup_verdict(backup, now)[1], "last_ok": iso(parse_moment(backup.get("last_ok"))),
+                       "age_hours": round(age, 1) if age is not None else None, "stale_after_hours": STALE_BACKUP_HOURS,
+                       "next": iso(parse_moment(backup.get("next"))), "last_result": backup.get("last_result")},
+            "system": c.health or {}, "schedule": _schedule_view(plan, now),
+            "updates": [_run_view(r) for r in runs]}
+
+
+async def update_home_assistant(entity_id: str) -> dict:
+    """Install one Core, OS or add-on update, as the page's Install does: Steward
+    takes a backup, confirms it landed, installs, and follows it to its end.
+
+    entity_id: from get_home_assistant's `installable`.
+
+    An update restarts Home Assistant or the add-on. An OS update reads as done
+    once it's written, and takes effect after a reboot, which Steward leaves to a
+    person. Refused while another is under way, or if nothing is waiting there.
+    Admin only, as on the page. Recorded in the audit log.
+    """
+    caller = current_caller()
+    try:
+        run = await start_update(mcp_app(), entity_id, mcp_username(caller.username))
+    except CannotStart as exc:
+        raise ValueError(f"Nothing was started: {exc}.") from None
+    await audit_mcp("homeassistant.update_started", "homeassistant_update", run.id,
+                    {"entity_id": entity_id, "title": run.title, "to": run.to_version})
+    return {**_run_view(run), "next": "get_home_assistant follows it: `updates` shows its state."}
+
+
+async def set_home_assistant_schedule(enabled: bool, weekday: int = 5, at_time: str = "05:00",
+                                      time_zone: str = "UTC") -> dict:
+    """Set the weekly slot for installing what's waiting, or turn it off, as the page does.
+
+    weekday: 0 for Monday to 6 for Sunday. at_time: "HH:MM" in time_zone.
+
+    At each slot Steward installs add-ons, then Core, then the OS, one at a time,
+    each after a confirmed backup, stopping at the first failure. A Core update
+    restarts Home Assistant and an OS update reboots the machine, which Steward
+    does itself here, with nobody watching. The first slot is the next one from
+    now. Admin only, and audited.
+    """
+    try:
+        weekly.parse_at(at_time)
+    except ValueError:
+        raise ValueError(f"at_time is HH:MM, not {at_time!r}.") from None
+    if not 0 <= weekday <= 6:
+        raise ValueError("weekday is 0 for Monday to 6 for Sunday.")
+    if zone(time_zone) is timezone.utc and time_zone.upper() != "UTC":
+        raise ValueError(f"Unknown time zone {time_zone!r}.")
+    row = await schedule.save(mcp_app(), enabled=enabled, weekday=weekday, at_time=at_time, time_zone=time_zone)
+    await audit_mcp("homeassistant.schedule_saved", "homeassistant_schedule", row.id,
+                    {"enabled": row.enabled, "when": weekly.describe(row)})
+    return _schedule_view(row, now_utc())
+
+
+def overview_findings(c, latest_run, now, plan=None) -> list[dict]:
+    """A connection that isn't answering, a problem with the system, stale backups,
+    a newest update that failed, and a scheduled run that stopped. Updates waiting
+    aren't a problem to raise."""
+    if c is None:
+        return []
+    found = []
+    reach, says = connection_says(c, now)
+    if reach == "down":
+        found.append(finding("warning", "home_assistant_down", "Home Assistant",
+                             f"Home Assistant isn't answering: {says}", since=c.last_ok_at, next="get_home_assistant()"))
+    problems = problems_open(c.health)
+    if problems:
+        more = f" (and {len(problems) - 1} more)" if len(problems) > 1 else ""
+        found.append(finding("warning", "home_assistant_problem", "Home Assistant", f"{problems[0]['title']}{more}",
+                             next="get_home_assistant()"))
+    tone, backup_says = backup_verdict(c.backup, now)
+    if c.last_ok_at is not None and tone == "warn":
+        found.append(finding("warning", "home_assistant_backup", "Home Assistant", backup_says,
+                             since=parse_moment((c.backup or {}).get("last_ok")), next="get_home_assistant()"))
+    if latest_run is not None and latest_run.state == "failed":
+        found.append(finding("warning", "home_assistant_update_failed", latest_run.title,
+                             f"The update of {latest_run.title} failed: {latest_run.message}",
+                             since=latest_run.finished_at, next="get_home_assistant()"))
+    if plan is not None and plan.stopped:
+        found.append(finding("warning", "home_assistant_schedule_stopped", "Home Assistant",
+                             f"The scheduled update run stopped: {plan.stopped}", since=weekly.fired_slot(plan),
+                             next="get_home_assistant()"))
+    return found
+
+
+async def mcp_overview(db) -> list[dict]:
+    c = (await db.execute(select(HomeAssistantConnection))).scalars().first()
+    latest = (await db.execute(select(HomeAssistantUpdate).order_by(
+        HomeAssistantUpdate.started_at.desc()).limit(1))).scalars().first()
+    plan = (await db.execute(select(HomeAssistantSchedule))).scalars().first()
+    return overview_findings(c, latest, now_utc(), plan)
+
+
+def register_tools(register) -> None:
+    register(get_home_assistant, role=UserRole.viewer, read_only=True)
+    register(update_home_assistant, role=UserRole.admin, read_only=False)
+    register(set_home_assistant_schedule, role=UserRole.admin, read_only=False)

plugins/homeassistant/migrations/versions/homeassistant_001_connection.py

@@ -0,0 +1,59 @@
+"""Home Assistant: the connection and the updates Steward was asked to install
+
+Revision ID: homeassistant_001_connection
+Revises: (none — a branch off core via depends_on)
+Create Date: 2026-10-31
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "homeassistant_001_connection"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = "homeassistant"
+depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
+
+
+def upgrade() -> None:
+    op.create_table(
+        "homeassistant_connection",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("url", sa.String(500), nullable=False),
+        sa.Column("token", sa.Text, nullable=False),
+        sa.Column("webhook_id", sa.Text, nullable=False),
+        sa.Column("verify_tls", sa.Boolean, nullable=False, server_default=sa.true()),
+        sa.Column("refused", sa.Boolean, nullable=False, server_default=sa.false()),
+        sa.Column("last_polled_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_ok_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_error", sa.Text, nullable=True),
+        sa.Column("updates", sa.JSON, nullable=False, server_default="[]"),
+        sa.Column("backup", sa.JSON, nullable=False, server_default="{}"),
+        sa.Column("automation", sa.String(20), nullable=True),
+    )
+    op.create_table(
+        "homeassistant_updates",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("entity_id", sa.String(255), nullable=False),
+        sa.Column("title", sa.String(255), nullable=False),
+        sa.Column("kind", sa.String(10), nullable=False),
+        sa.Column("from_version", sa.String(100), nullable=True),
+        sa.Column("to_version", sa.String(100), nullable=True),
+        sa.Column("state", sa.String(20), nullable=False, server_default="backing_up"),
+        sa.Column("message", sa.Text, nullable=True),
+        sa.Column("started_by", sa.String(100), nullable=False, server_default=""),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("backup_before", sa.String(64), nullable=True),
+        sa.Column("backup_event_before", sa.String(64), nullable=True),
+        sa.Column("triggered_before", sa.String(64), nullable=True),
+        sa.Column("backup_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("install_sent_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.execute("CREATE UNIQUE INDEX uq_homeassistant_update_under_way ON homeassistant_updates ((true)) "
+               "WHERE state IN ('backing_up', 'installing')")
+
+
+def downgrade() -> None:
+    op.drop_table("homeassistant_updates")
+    op.drop_table("homeassistant_connection")

plugins/homeassistant/migrations/versions/homeassistant_002_sign_in.py

@@ -0,0 +1,42 @@
+"""Home Assistant: Steward holds a sign-in, in place of a token and a webhook
+
+Revision ID: homeassistant_002_sign_in
+Revises: homeassistant_001_connection
+Create Date: 2026-10-31
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "homeassistant_002_sign_in"
+down_revision: Union[str, None] = "homeassistant_001_connection"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # A saved token can't be refreshed like a sign-in, so that connection signs in again.
+    op.execute("DELETE FROM homeassistant_connection")
+    op.drop_column("homeassistant_connection", "webhook_id")
+    op.drop_column("homeassistant_connection", "automation")
+    op.drop_column("homeassistant_updates", "triggered_before")
+    op.alter_column("homeassistant_connection", "token", new_column_name="refresh_token")
+    op.add_column("homeassistant_connection", sa.Column("client_id", sa.String(500), nullable=False))
+    op.add_column("homeassistant_connection",
+                  sa.Column("can_install", sa.Boolean, nullable=False, server_default=sa.false()))
+    op.add_column("homeassistant_connection",
+                  sa.Column("signed_in_by", sa.String(100), nullable=False, server_default=""))
+    op.add_column("homeassistant_connection", sa.Column("signed_in_at", sa.DateTime(timezone=True), nullable=True))
+
+
+def downgrade() -> None:
+    op.execute("DELETE FROM homeassistant_connection")
+    op.drop_column("homeassistant_connection", "signed_in_at")
+    op.drop_column("homeassistant_connection", "signed_in_by")
+    op.drop_column("homeassistant_connection", "can_install")
+    op.drop_column("homeassistant_connection", "client_id")
+    op.alter_column("homeassistant_connection", "refresh_token", new_column_name="token")
+    op.add_column("homeassistant_updates", sa.Column("triggered_before", sa.String(64), nullable=True))
+    op.add_column("homeassistant_connection", sa.Column("automation", sa.String(20), nullable=True))
+    op.add_column("homeassistant_connection", sa.Column("webhook_id", sa.Text, nullable=False))

plugins/homeassistant/migrations/versions/homeassistant_003_health.py

@@ -0,0 +1,37 @@
+"""Home Assistant: what the poll reads of the system, and a weekly slot for updates
+
+Revision ID: homeassistant_003_health
+Revises: homeassistant_002_sign_in
+Create Date: 2026-11-01
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "homeassistant_003_health"
+down_revision: Union[str, None] = "homeassistant_002_sign_in"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("homeassistant_connection",
+                  sa.Column("health", sa.JSON, nullable=False, server_default="{}"))
+    op.create_table(
+        "homeassistant_schedule",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("enabled", sa.Boolean, nullable=False),
+        sa.Column("weekday", sa.Integer, nullable=False),
+        sa.Column("at_time", sa.String(5), nullable=False),
+        sa.Column("time_zone", sa.String(64), nullable=False),
+        sa.Column("window_minutes", sa.Integer, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("last_slot_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("stopped", sa.Text, nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("homeassistant_schedule")
+    op.drop_column("homeassistant_connection", "health")

plugins/homeassistant/migrations/versions/homeassistant_004_reboot.py

@@ -0,0 +1,34 @@
+"""Home Assistant: an update the schedule installs can reboot the machine and follow it
+
+Revision ID: homeassistant_004_reboot
+Revises: homeassistant_003_health
+Create Date: 2026-11-02
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "homeassistant_004_reboot"
+down_revision: Union[str, None] = "homeassistant_003_health"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _under_way(states: str) -> None:
+    op.execute("DROP INDEX uq_homeassistant_update_under_way")
+    op.execute("CREATE UNIQUE INDEX uq_homeassistant_update_under_way ON homeassistant_updates ((true)) "
+               f"WHERE state IN ({states})")
+
+
+def upgrade() -> None:
+    op.add_column("homeassistant_updates",
+                  sa.Column("reboot_after", sa.Boolean, nullable=False, server_default=sa.false()))
+    op.add_column("homeassistant_updates", sa.Column("reboot_sent_at", sa.DateTime(timezone=True), nullable=True))
+    _under_way("'backing_up', 'installing', 'rebooting'")
+
+
+def downgrade() -> None:
+    _under_way("'backing_up', 'installing'")
+    op.drop_column("homeassistant_updates", "reboot_sent_at")
+    op.drop_column("homeassistant_updates", "reboot_after")

plugins/homeassistant/models.py

@@ -0,0 +1,92 @@
+# plugins/homeassistant/models.py
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from sqlalchemy import JSON, Boolean, DateTime, Index, Integer, String, Text, text
+from sqlalchemy.orm import Mapped, mapped_column
+
+from steward.core.ids import time_ordered_id
+from steward.models.base import Base
+
+
+class HomeAssistantConnection(Base):
+    """The Home Assistant instance Steward looks after; at most one row.
+
+    `refresh_token` is what Home Assistant's sign-in gave Steward, encrypted. It
+    has the same access as the user who signed in, whatever that is: Home
+    Assistant has no narrower permission. `can_install` is whether that user was
+    an administrator, which backups need. The last_* columns and `updates` and
+    `backup` and `health` are the newest poll's outcome, which the pages read. `refused` stops
+    all polling until someone signs in again or tries again: Home Assistant bans
+    an address after a few refused requests.
+    """
+    __tablename__ = "homeassistant_connection"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
+    url: Mapped[str] = mapped_column(String(500), nullable=False)
+    client_id: Mapped[str] = mapped_column(String(500), nullable=False)
+    refresh_token: Mapped[str] = mapped_column(Text, nullable=False)
+    can_install: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    signed_in_by: Mapped[str] = mapped_column(String(100), nullable=False, default="")
+    signed_in_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    verify_tls: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    refused: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    last_polled_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    last_ok_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    updates: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
+    backup: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
+    health: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
+
+
+class HomeAssistantSchedule(Base):
+    """The weekly slot for installing what's waiting; at most one row, none until an
+    admin sets it. `last_slot_at` is the slot that last fired, and `stopped` says why
+    that slot ended early. Times are read through steward.core.weekly."""
+    __tablename__ = "homeassistant_schedule"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
+    enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    weekday: Mapped[int] = mapped_column(Integer, nullable=False)
+    at_time: Mapped[str] = mapped_column(String(5), nullable=False)
+    time_zone: Mapped[str] = mapped_column(String(64), nullable=False)
+    window_minutes: Mapped[int] = mapped_column(Integer, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+    last_slot_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    stopped: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class HomeAssistantUpdate(Base):
+    """One update Steward was asked to install, from the click to its end. It
+    moves through backing_up and installing to done or failed, and its row
+    carries everything needed to carry on after Steward restarts. The *_before
+    columns are Home Assistant's own readings at the start, so "a newer backup"
+    is judged on its clock alone. One can be under way at a time. `reboot_after`
+    sends it through rebooting too, for an OS update the schedule installs.
+    """
+    __tablename__ = "homeassistant_updates"
+    __table_args__ = (
+        Index("uq_homeassistant_update_under_way", text("(true)"), unique=True,
+              postgresql_where=text("state IN ('backing_up', 'installing', 'rebooting')")),
+    )
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
+    entity_id: Mapped[str] = mapped_column(String(255), nullable=False)
+    title: Mapped[str] = mapped_column(String(255), nullable=False)
+    kind: Mapped[str] = mapped_column(String(10), nullable=False)
+    from_version: Mapped[str | None] = mapped_column(String(100), nullable=True)
+    to_version: Mapped[str | None] = mapped_column(String(100), nullable=True)
+    state: Mapped[str] = mapped_column(String(20), nullable=False, default="backing_up")
+    message: Mapped[str | None] = mapped_column(Text, nullable=True)
+    started_by: Mapped[str] = mapped_column(String(100), nullable=False, default="")
+    started_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+    backup_before: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    backup_event_before: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    backup_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    install_sent_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    reboot_after: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    reboot_sent_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

plugins/homeassistant/plugin.yaml

@@ -0,0 +1,17 @@
+# plugins/homeassistant/plugin.yaml
+name: homeassistant
+version: "1.0.0"
+description: "Keeps Home Assistant running: shows whether backups are happening and whether the system has a problem, and installs Core, OS and add-on updates after a confirmed backup, on a click or a weekly schedule."
+author: "Steward"
+license: "MIT"
+min_app_version: "0.1.0"
+repository_url: "https://git.fabledsword.com/bvandeusen/FabledSteward"
+homepage: "https://git.fabledsword.com/bvandeusen/FabledSteward/src/branch/dev/plugins/homeassistant"
+tags:
+  - home-assistant
+  - updates
+  - backups
+
+# The connection is added and edited on the Home Assistant page
+# (/plugins/homeassistant/).
+config: {}

plugins/homeassistant/routes.py

@@ -0,0 +1,249 @@
+"""The Home Assistant page. Every GET reads Steward's own records, which the poll
+fills. Only signing in, removing the connection and Install, an admin's explicit
+actions, ask Home Assistant anything (System 178's ruling)."""
+from __future__ import annotations
+
+import hmac
+import logging
+import secrets
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+from quart import Blueprint, current_app, redirect, render_template, request, session
+from sqlalchemy import select
+
+from steward.auth.middleware import require_role
+from steward.core import weekly
+from steward.core.audit import log_audit
+from steward.core.crypto import encrypt_secret
+from steward.core.formatting import zone
+from steward.core.settings import public_base_url
+from steward.models.users import User, UserRole
+
+from . import client, schedule
+from .models import HomeAssistantConnection, HomeAssistantSchedule, HomeAssistantUpdate
+from .service import STALE_BACKUP_HOURS, backup_age_hours, backup_verdict, installable, parse_moment, problems_open
+from .updates import UNDER_WAY, CannotStart, start_update
+
+logger = logging.getLogger(__name__)
+
+homeassistant_bp = Blueprint("homeassistant", __name__, template_folder="templates")
+
+RECENT_UPDATES = 10
+_SIGN_IN = "homeassistant_sign_in"      # in the session between leaving for Home Assistant and coming back
+
+
+def _actor() -> tuple[str | None, str]:
+    return session.get("user_id"), session.get("username", "")
+
+
+async def _connection(db) -> HomeAssistantConnection | None:
+    return (await db.execute(select(HomeAssistantConnection))).scalars().first()
+
+
+def connection_says(connection, now: datetime) -> tuple[str, str]:
+    """(badge state, sentence) for whether Home Assistant is answering."""
+    if connection.refused:
+        return "down", f"{connection.last_error} Steward has stopped asking until you sign in again or try again."
+    if connection.last_polled_at is None:
+        return "pending", "Not read yet: the first read runs within seconds."
+    if connection.last_error:
+        return "down", connection.last_error
+    return "up", "Answering."
+
+
+async def _render_index(error: str | None = None, status: int = 200):
+    async with current_app.db_sessionmaker() as db:
+        connection = await _connection(db)
+        runs = list((await db.execute(select(HomeAssistantUpdate).order_by(
+            HomeAssistantUpdate.started_at.desc()).limit(RECENT_UPDATES))).scalars())
+        plan = (await db.execute(select(HomeAssistantSchedule))).scalars().first()
+        me = await db.get(User, session.get("user_id")) if session.get("user_id") else None
+    if connection is None:
+        return await render_template("homeassistant/index.html", connection=None, error=error), status
+    default_zone = (me.time_zone if me and me.time_zone else "UTC")
+    now = datetime.now(timezone.utc)
+    reach, says = connection_says(connection, now)
+    updates = connection.updates or []
+    backup = connection.backup or {}
+    tone, backup_says = backup_verdict(backup, now)
+    return await render_template(
+        "homeassistant/index.html", connection=connection, error=error, reach=reach, says=says,
+        installable=installable(connection),
+        others=[u for u in updates if u["waiting"] and not u["kind"]],
+        current=[u for u in updates if not u["waiting"] and u["kind"]],
+        runs=runs, under_way=next((r for r in runs if r.state in UNDER_WAY), None),
+        backup=backup, backup_tone=tone, backup_says=backup_says,
+        backup_at=parse_moment(backup.get("last_ok")), backup_next=parse_moment(backup.get("next")),
+        backup_age=backup_age_hours(backup, now), stale_hours=STALE_BACKUP_HOURS,
+        health=connection.health or {}, problems=problems_open(connection.health),
+        plan=plan, plan_when=weekly.describe(plan) if plan else None,
+        plan_next=weekly.next_slot(plan, now) if plan and plan.enabled else None,
+        weekdays=weekly.WEEKDAYS, default_zone=default_zone), status
+
+
+@homeassistant_bp.get("/")
+@require_role(UserRole.viewer)
+async def index():
+    return await _render_index()
+
+
+@homeassistant_bp.post("/install")
+@require_role(UserRole.admin)
+async def install():
+    form = await request.form
+    entity_id = (form.get("entity_id") or "").strip()
+    user_id, username = _actor()
+    try:
+        run = await start_update(current_app, entity_id, username)
+    except CannotStart as exc:
+        return await _render_index(f"Nothing was started: {exc}.", 409)
+    await log_audit(current_app, user_id, username, "homeassistant.update_started",
+                    entity_type="homeassistant_update", entity_id=run.id,
+                    detail={"entity_id": entity_id, "title": run.title, "to": run.to_version})
+    return redirect("/plugins/homeassistant/")
+
+
+@homeassistant_bp.post("/schedule")
+@require_role(UserRole.admin)
+async def save_schedule():
+    """Set the weekly slot for installing what's waiting, or turn it off."""
+    form = await request.form
+    time_zone = (form.get("time_zone") or "UTC").strip()
+    try:
+        weekday, at_time = int(form.get("weekday", "")), (form.get("at_time") or "").strip()
+        weekly.parse_at(at_time)
+    except ValueError:
+        return await _render_index("Choose a day and a time.", 422)
+    if not 0 <= weekday <= 6 or (zone(time_zone) is timezone.utc and time_zone.upper() != "UTC"):
+        return await _render_index(f"Choose a day, and a known time zone rather than “{time_zone}”.", 422)
+    row = await schedule.save(current_app, enabled=bool(form.get("enabled")), weekday=weekday,
+                              at_time=at_time, time_zone=time_zone)
+    await log_audit(current_app, *_actor(), "homeassistant.schedule_saved", entity_type="homeassistant_schedule",
+                    entity_id=row.id, detail={"enabled": row.enabled, "when": weekly.describe(row)})
+    return redirect("/plugins/homeassistant/")
+
+
+# ── The connection ───────────────────────────────────────────────────────────
+
+async def _render_form(connection, url: str | None = None, verify_tls: bool | None = None,
+                       error: str | None = None, status: int = 200):
+    return await render_template(
+        "homeassistant/form.html", connection=connection, error=error,
+        url=url if url is not None else (connection.url if connection else ""),
+        verify_tls=verify_tls if verify_tls is not None else (connection.verify_tls if connection else True)), status
+
+
+@homeassistant_bp.get("/connection")
+@require_role(UserRole.admin)
+async def connection_form():
+    async with current_app.db_sessionmaker() as db:
+        connection = await _connection(db)
+    return await _render_form(connection)
+
+
+@homeassistant_bp.post("/connection/sign-in")
+@require_role(UserRole.admin)
+async def sign_in():
+    """Send the browser to Home Assistant's own sign-in page."""
+    form = await request.form
+    url = (form.get("url") or "").strip().rstrip("/")
+    verify_tls = bool(form.get("verify_tls"))
+    if not url.startswith(("http://", "https://")):
+        async with current_app.db_sessionmaker() as db:
+            connection = await _connection(db)
+        return await _render_form(connection, url, verify_tls, "Enter Home Assistant's address: http://… or https://…", 422)
+    base, state = public_base_url(request), secrets.token_urlsafe(32)
+    client_id = f"{base}/"
+    session[_SIGN_IN] = {"state": state, "url": url, "verify_tls": verify_tls, "client_id": client_id}
+    return redirect(client.sign_in_url(url, client_id, f"{base}/plugins/homeassistant/connection/callback", state))
+
+
+@homeassistant_bp.get("/connection/callback")
+@require_role(UserRole.admin)
+async def sign_in_callback():
+    """Where Home Assistant sends the browser back. A GET that asks Home Assistant
+    something, because the admin's sign-in has only just happened and its code
+    lasts minutes. The state shows the sign-in began here, for this admin."""
+    pending = session.pop(_SIGN_IN, None)
+    state, code = request.args.get("state", ""), request.args.get("code", "")
+    async with current_app.db_sessionmaker() as db:
+        connection = await _connection(db)
+    if not pending or not hmac.compare_digest(pending["state"].encode(), state.encode()):
+        return await _render_form(connection, error="That sign-in didn't start from this page, so it wasn't used. "
+                                  "If it keeps happening, check Public base URL in Settings → General: "
+                                  "Steward must be opened at that address.", status=400)
+    url, verify_tls, client_id = pending["url"], pending["verify_tls"], pending["client_id"]
+    if not code:
+        return await _render_form(connection, url, verify_tls, "Home Assistant didn't finish the sign-in. Nothing changed.", 400)
+    try:
+        refresh_token, token = await client.redeem_code(url, client_id, code, verify_tls)
+        can_install = await client.is_admin(url, token, verify_tls)
+    except (client.Refused, client.Unreachable) as exc:
+        return await _render_form(connection, url, verify_tls, f"The sign-in couldn't be finished: {exc}", 502)
+    user_id, username = _actor()
+    previous = None
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            connection = await _connection(db)
+            if connection is None:
+                connection = HomeAssistantConnection(url=url, client_id=client_id, refresh_token="")
+                db.add(connection)
+            else:
+                previous = (connection.url, connection.client_id, connection.refresh_token, connection.verify_tls)
+            connection.url, connection.client_id, connection.verify_tls = url, client_id, verify_tls
+            connection.refresh_token = encrypt_secret(refresh_token)
+            connection.can_install, connection.signed_in_by = can_install, username
+            connection.signed_in_at = datetime.now(timezone.utc)
+            connection.refused, connection.last_error, connection.last_polled_at = False, None, None
+            await db.flush()
+            connection_id = connection.id
+    if previous is not None:
+        await _end_sign_in(SimpleNamespace(url=previous[0], client_id=previous[1], refresh_token=previous[2],
+                                           verify_tls=previous[3]))
+    await log_audit(current_app, user_id, username, "homeassistant.connection_saved",
+                    entity_type="homeassistant_connection", entity_id=connection_id,
+                    detail={"name": url, "can_install": can_install})
+    return redirect("/plugins/homeassistant/")
+
+
+async def _end_sign_in(connection) -> bool:
+    """End a sign-in Steward no longer needs, so it doesn't stay valid on Home
+    Assistant. It can be ended there by hand too, so failing here stops nothing."""
+    try:
+        await client.revoke(connection)
+    except (client.Refused, client.Unreachable) as exc:
+        logger.warning("Home Assistant: couldn't end Steward's old sign-in: %s", exc)
+        return False
+    return True
+
+
+@homeassistant_bp.post("/connection/retry")
+@require_role(UserRole.admin)
+async def retry_connection():
+    """Ask again after a refusal, once a ban has been dealt with."""
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            connection = await _connection(db)
+            if connection is not None:
+                connection.refused, connection.last_polled_at = False, None
+    return redirect("/plugins/homeassistant/")
+
+
+@homeassistant_bp.post("/connection/delete")
+@require_role(UserRole.admin)
+async def delete_connection():
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            connection = await _connection(db)
+            if connection is None:
+                return redirect("/plugins/homeassistant/")
+            connection_id, url = connection.id, connection.url
+            await db.delete(connection)
+    ended = await _end_sign_in(connection)
+    await log_audit(current_app, *_actor(), "homeassistant.connection_removed",
+                    entity_type="homeassistant_connection", entity_id=connection_id, detail={"name": url, "sign_in_ended": ended})
+    if not ended:
+        return await _render_index("Removed from Steward, but Home Assistant couldn't be reached to end the sign-in. "
+                                   "End it yourself under Security on that user's profile in Home Assistant.")
+    return redirect("/plugins/homeassistant/")

plugins/homeassistant/schedule.py

@@ -0,0 +1,95 @@
+# plugins/homeassistant/schedule.py
+"""Installing what's waiting at a weekly slot: one at a time through the same path
+as the page's Install, each after its own confirmed backup. Add-ons go first and
+the OS last, as it reboots the machine. The first failure ends the slot, leaving the
+rest for someone to look at. Everything is on the rows, so a restart of Steward or
+of Home Assistant only delays the next step."""
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import select
+
+from steward.core import weekly
+
+from .models import HomeAssistantConnection, HomeAssistantSchedule, HomeAssistantUpdate
+from .service import installable
+from .updates import CannotStart, start_update, under_way
+
+ACTOR = "schedule"
+WINDOW_MINUTES = 240
+_ORDER = {"addon": 0, "core": 1, "os": 2}
+
+
+def next_to_install(waiting: list[dict], tried: set[str]) -> dict | None:
+    left = [u for u in waiting if u["entity_id"] not in tried and not u["in_progress"]]
+    return min(left, key=lambda u: _ORDER[u["kind"]], default=None)
+
+
+async def _stop(app, schedule_id: str, why: str) -> None:
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            row = await session.get(HomeAssistantSchedule, schedule_id)
+            if row is not None:
+                row.stopped = why
+
+
+async def _fire(app, schedule_id: str, slot: datetime) -> None:
+    """Open the slot, and have the next poll say what's waiting at this moment."""
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            row = await session.get(HomeAssistantSchedule, schedule_id)
+            row.last_slot_at, row.stopped = slot, None
+            connection = (await session.execute(select(HomeAssistantConnection))).scalars().first()
+            if connection is not None:
+                connection.last_polled_at = None
+
+
+async def step(app, transport=None) -> None:
+    """Fire a slot that has come, or start the next update of one that's open."""
+    now = datetime.now(timezone.utc)
+    async with app.db_sessionmaker() as session:
+        schedule = (await session.execute(select(HomeAssistantSchedule))).scalars().first()
+        connection = (await session.execute(select(HomeAssistantConnection))).scalars().first()
+        if schedule is None or connection is None:
+            return
+        slot = weekly.is_due(schedule, now)
+        if slot is not None:
+            await _fire(app, schedule.id, slot)
+            return
+        slot = weekly.fired_slot(schedule)
+        if not schedule.enabled or schedule.stopped or slot is None \
+                or now - slot >= timedelta(minutes=schedule.window_minutes):
+            return
+        # Only on a fresh, good read: Home Assistant may be restarting from the last update.
+        if connection.refused or connection.last_polled_at is None or connection.last_error:
+            return
+        if await under_way(session) is not None:
+            return
+        runs = (await session.execute(select(HomeAssistantUpdate).where(
+            HomeAssistantUpdate.started_at >= slot))).scalars().all()
+    failed = next((r for r in runs if r.state == "failed"), None)
+    if failed is not None:
+        await _stop(app, schedule.id, f"{failed.title}: {failed.message} Nothing more was installed in this slot.")
+        return
+    update = next_to_install(installable(connection), {r.entity_id for r in runs})
+    if update is None:
+        return
+    try:
+        await start_update(app, update["entity_id"], ACTOR, transport, reboot_after=update["kind"] == "os")
+    except CannotStart as exc:
+        await _stop(app, schedule.id, f"{update['title']} wasn't started: {exc}. Nothing more was installed in this slot.")
+
+
+async def save(app, *, enabled: bool, weekday: int, at_time: str, time_zone: str) -> HomeAssistantSchedule:
+    """Set the slot. Its first is the next one from now, not one already past."""
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            row = (await session.execute(select(HomeAssistantSchedule))).scalars().first()
+            if row is None:
+                row = HomeAssistantSchedule(window_minutes=WINDOW_MINUTES)
+                session.add(row)
+            row.enabled, row.weekday, row.time_zone = enabled, weekday, time_zone
+            row.at_time = f"{weekly.parse_at(at_time):%H:%M}"
+            row.last_slot_at, row.stopped = datetime.now(timezone.utc), None
+    return row

plugins/homeassistant/scheduler.py

@@ -0,0 +1,116 @@
+# plugins/homeassistant/scheduler.py
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING
+
+from sqlalchemy import select
+
+from steward.core.scheduler import ScheduledTask
+
+from . import client, schedule
+from .models import HomeAssistantConnection, HomeAssistantSchedule, HomeAssistantUpdate
+from .service import RESOURCE, SOURCE, backup_age_hours, installable, read_health, read_snapshot
+from .updates import advance
+
+if TYPE_CHECKING:
+    from quart import Quart
+
+logger = logging.getLogger(__name__)
+
+# An update under way is followed each tick, with one small read of its entity.
+TICK_SECONDS = 15
+# The whole entity list is a few hundred KB, and updates and backups change rarely.
+POLL_SECONDS = 300
+# A step past this is given up until the next tick, whatever the client's own
+# timeouts did: a slow drip of bytes can keep each read inside its timeout for ever.
+STEP_DEADLINE_SECONDS = 60
+
+
+def make_tick_task(app: "Quart") -> ScheduledTask:
+    busy = asyncio.Lock()
+
+    async def tick() -> None:
+        if busy.locked():
+            return
+        async with busy:
+            for step in (poll, advance, schedule.step):
+                try:
+                    await asyncio.wait_for(step(app), STEP_DEADLINE_SECONDS)
+                except TimeoutError:
+                    logger.warning("Home Assistant %s gave up after %d s", step.__name__, STEP_DEADLINE_SECONDS)
+
+    return ScheduledTask(name="homeassistant_tick", coro_factory=tick,
+                         interval_seconds=TICK_SECONDS, run_on_startup=True)
+
+
+def is_due(connection, now: datetime) -> bool:
+    """Whether to poll. Never after a refusal: another refused request counts
+    toward Home Assistant banning this address."""
+    if connection.refused:
+        return False
+    if connection.last_polled_at is None:
+        return True
+    return (now - connection.last_polled_at).total_seconds() >= POLL_SECONDS - TICK_SECONDS / 2
+
+
+async def update_failed(session) -> bool:
+    """Whether the newest update failed, or the schedule couldn't start one."""
+    stopped = (await session.execute(select(HomeAssistantSchedule.stopped))).scalars().first()
+    newest = (await session.execute(select(HomeAssistantUpdate.state).order_by(
+        HomeAssistantUpdate.started_at.desc()).limit(1))).scalars().first()
+    return bool(stopped) or newest == "failed"
+
+
+async def poll(app: "Quart", transport=None, connect=None) -> None:
+    from steward.core.alerts import MetricBatch
+
+    now = datetime.now(timezone.utc)
+    async with app.db_sessionmaker() as session:
+        connection = (await session.execute(select(HomeAssistantConnection))).scalars().first()
+    if connection is None or not is_due(connection, now):
+        return
+    snapshot, health, error, refused = None, None, None, False
+    try:
+        token = await client.access_token(connection, transport)
+        snapshot = read_snapshot(await client.read_states(connection.url, token, connection.verify_tls, transport))
+        try:
+            health = read_health(await client.read_system(connection.url, token, connection.verify_tls, connect))
+        except client.Unreachable as exc:
+            # Updates and backups were read, so it's answering. The last report stands, and says it's old.
+            health = {**(connection.health or {}), "error": str(exc)}
+    except client.Refused as exc:
+        snapshot, error, refused = None, str(exc), True
+    except client.Unreachable as exc:
+        error = str(exc)
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            row = await session.get(HomeAssistantConnection, connection.id)
+            if row is None:         # removed while it was being polled
+                return
+            batch = MetricBatch()
+            row.last_polled_at, row.last_error, row.refused = now, error, refused
+            if snapshot is None:
+                batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="is_up", value=0.0, recorded_at=now)
+            else:
+                row.last_ok_at = now
+                row.updates, row.backup, row.health = snapshot["updates"], snapshot["backup"], health
+                batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="is_up", value=1.0, recorded_at=now)
+                batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="updates_waiting",
+                          value=len(installable(row)), recorded_at=now)
+                age = backup_age_hours(snapshot["backup"], now)
+                if age is not None:
+                    batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="backup_age_hours",
+                              value=age, recorded_at=now)
+                if not health.get("error"):
+                    if health["repairs_read"]:
+                        batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="problems_open",
+                                  value=len(health["problems"]), recorded_at=now)
+                    if health["disk_free_pct"] is not None:
+                        batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="disk_free_pct",
+                                  value=health["disk_free_pct"], recorded_at=now)
+                batch.add(source_module=SOURCE, resource_name=RESOURCE, metric_name="update_failed",
+                          value=1.0 if await update_failed(session) else 0.0, recorded_at=now)
+            await batch.record(session)

plugins/homeassistant/service.py

@@ -0,0 +1,161 @@
+# plugins/homeassistant/service.py
+"""Reading what Home Assistant reports: which updates it knows of, which of them
+Steward may install, how its backups stand, and whether the system has a problem."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+SOURCE = "homeassistant"
+RESOURCE = "homeassistant"
+# Backups run daily, so past a day and a bit one has been missed.
+STALE_BACKUP_HOURS = 26
+LOW_DISK_PCT = 10
+
+CORE = "update.home_assistant_core_update"
+OS = "update.home_assistant_operating_system_update"
+SUPERVISOR = "update.home_assistant_supervisor_update"
+LAST_BACKUP = "sensor.backup_last_successful_automatic_backup"
+NEXT_BACKUP = "sensor.backup_next_scheduled_automatic_backup"
+BACKUP_MANAGER = "sensor.backup_backup_manager_state"
+BACKUP_EVENT = "event.backup_automatic_backup"
+
+_KIND_WORDS = {"core": "Core", "os": "Operating system", "addon": "Add-on"}
+_KIND_ORDER = {"core": 0, "os": 1, "addon": 2, None: 3}
+
+def classify(entity: dict) -> str | None:
+    """"core", "os" or "addon" for an update Steward may install; None for the
+    rest: Supervisor updates itself, and firmware and custom integrations are
+    Home Assistant's own to offer."""
+    entity_id = entity.get("entity_id")
+    if entity_id == CORE:
+        return "core"
+    if entity_id == OS:
+        return "os"
+    if entity_id == SUPERVISOR:
+        return None
+    picture = (entity.get("attributes") or {}).get("entity_picture") or ""
+    return "addon" if picture.startswith("/api/hassio/addons/") else None
+
+
+def _update(entity: dict) -> dict:
+    a = entity.get("attributes") or {}
+    kind = classify(entity)
+    url = a.get("release_url") or ""
+    return {"entity_id": entity["entity_id"], "kind": kind, "kind_word": _KIND_WORDS.get(kind, "Other"),
+            "title": a.get("title") or a.get("friendly_name") or entity["entity_id"],
+            "installed": a.get("installed_version"), "latest": a.get("latest_version"),
+            "waiting": entity.get("state") == "on", "in_progress": bool(a.get("in_progress")),
+            "auto_update": bool(a.get("auto_update")),
+            "release_url": url if url.startswith(("http://", "https://")) else None}
+
+
+def _state(entity: dict | None) -> str | None:
+    value = (entity or {}).get("state")
+    return None if value in (None, "unknown", "unavailable") else value
+
+
+def parse_moment(value: str | None) -> datetime | None:
+    try:
+        moment = datetime.fromisoformat(value) if value else None
+    except ValueError:
+        return None
+    if moment is not None and moment.tzinfo is None:
+        moment = moment.replace(tzinfo=timezone.utc)
+    return moment
+
+
+def is_newer(value: str | None, than: str | None) -> bool:
+    """Whether reading `value` is a later moment than `than`, where `than` may be never."""
+    moment, before = parse_moment(value), parse_moment(than)
+    return moment is not None and (before is None or moment > before)
+
+
+def read_snapshot(states: list[dict]) -> dict:
+    """What a poll keeps of Home Assistant's entities."""
+    by_id = {e.get("entity_id"): e for e in states}
+    updates = [_update(e) for i, e in by_id.items() if (i or "").startswith("update.") and e.get("state") != "unavailable"]
+    updates.sort(key=lambda u: (not u["waiting"], _KIND_ORDER[u["kind"]], u["title"].lower()))
+    event = by_id.get(BACKUP_EVENT) or {}
+    event_attrs = event.get("attributes") or {}
+    return {
+        "updates": updates,
+        "backup": {"last_ok": _state(by_id.get(LAST_BACKUP)), "next": _state(by_id.get(NEXT_BACKUP)),
+                   "manager": _state(by_id.get(BACKUP_MANAGER)), "last_result": event_attrs.get("event_type"),
+                   "failed_reason": event_attrs.get("failed_reason"), "known": LAST_BACKUP in by_id},
+    }
+
+
+def backup_age_hours(backup: dict | None, now: datetime) -> float | None:
+    last = parse_moment((backup or {}).get("last_ok"))
+    return None if last is None else max(0.0, (now - last).total_seconds() / 3600)
+
+
+def installable(connection) -> list[dict]:
+    """The waiting updates Steward may install."""
+    return [u for u in (connection.updates or []) if u["waiting"] and u["kind"]]
+
+
+def backup_verdict(backup: dict | None, now: datetime) -> tuple[str, str]:
+    """(tone, sentence) for how the backups stand."""
+    backup = backup or {}
+    if not backup.get("known"):
+        return "warn", "Home Assistant reports no automatic backups. Turn them on under Settings → System → Backups."
+    age = backup_age_hours(backup, now)
+    if age is None:
+        return "warn", "Home Assistant has no successful automatic backup yet."
+    if backup.get("last_result") == "failed":
+        reason = backup.get("failed_reason")
+        return "warn", f"The newest backup attempt failed{f' ({reason})' if reason else ''}."
+    if age > STALE_BACKUP_HOURS:
+        return "warn", f"The last good backup is {age:.0f} hours old, past the {STALE_BACKUP_HOURS} hours expected."
+    return "ok", "Backups are up to date."
+
+
+def _issue_title(issue: dict, titles: dict) -> str:
+    key = issue.get("translation_key") or issue.get("issue_id") or ""
+    title = titles.get(f"component.{issue.get('domain')}.issues.{key}.title")
+    if not title:
+        return key.removeprefix("issue_").replace("_", " ").capitalize() or "A repair is open"
+    for name, value in (issue.get("translation_placeholders") or {}).items():
+        title = title.replace("{" + name + "}", str(value))
+    return title
+
+
+def read_health(found: dict) -> dict:
+    """What a poll keeps of client.read_system: the problems to show, and the disk.
+    A part Home Assistant wouldn't answer is left out rather than read as fine:
+    `repairs_read` says whether the repairs were seen, and `supervised` whether a
+    Supervisor answered, as it doesn't on every install or for every user."""
+    problems = []
+    if (found.get("config") or {}).get("recovery_mode"):
+        problems.append({"kind": "recovery", "severity": "critical",
+                         "title": "Home Assistant started in recovery mode: its configuration didn't load."})
+    issues = found.get("issues")
+    for issue in issues or []:
+        if not issue.get("ignored") and not issue.get("dismissed_version"):
+            problems.append({"kind": "repair", "severity": issue.get("severity") or "warning",
+                             "title": _issue_title(issue, found.get("titles") or {})})
+    resolution = found.get("resolution") or {}
+    for word in ("unhealthy", "unsupported"):
+        for reason in resolution.get(word) or []:
+            problems.append({"kind": "supervisor", "severity": "error" if word == "unhealthy" else "warning",
+                             "title": f"The Supervisor calls this system {word}: {str(reason).replace('_', ' ')}."})
+    host = found.get("host") or {}
+    free, used, total = host.get("disk_free"), host.get("disk_used"), host.get("disk_total")
+    free_pct = round(100 * free / total, 1) if free is not None and total else None
+    # Used and free don't add up to the total: the system reserves some of the disk.
+    used_pct = round(100 * used / total, 1) if used is not None and total else None
+    if free_pct is not None and free_pct < LOW_DISK_PCT:
+        problems.append({"kind": "disk", "severity": "error", "title": f"Only {free:g} GB of {total:g} GB free."})
+    boots = found.get("boots") or {}
+    for addon in found.get("addons") or []:
+        if boots.get(addon.get("slug")) == "auto":
+            problems.append({"kind": "addon", "severity": "error",
+                             "title": f"{addon.get('name') or addon.get('slug')} should start on its own and isn't running."})
+    return {"repairs_read": issues is not None, "supervised": found.get("resolution") is not None, "problems": problems,
+            "disk_free_gb": free, "disk_used_gb": used, "disk_total_gb": total,
+            "disk_free_pct": free_pct, "disk_used_pct": used_pct}
+
+
+def problems_open(health: dict | None) -> list[dict]:
+    return (health or {}).get("problems") or []

plugins/homeassistant/templates/homeassistant/form.html

@@ -0,0 +1,50 @@
+{# plugins/homeassistant/templates/homeassistant/form.html — where Home Assistant
+   is, what signing in gives Steward, and leaving for its sign-in page. #}
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs, page_header %}
+{% block title %}Home Assistant connection — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Home Assistant", "/plugins/homeassistant/"), ("Connection", "")]) }}{% endblock %}
+{% block content %}
+{{ page_header("Home Assistant connection") }}
+
+<div class="form-page">
+{% if error %}<p class="field-error" role="alert">{{ error }}</p>{% endif %}
+{% if connection %}
+<p class="field-hint">Signed in{% if connection.signed_in_at %} {{ connection.signed_in_at|ago }}{% endif %}{% if connection.signed_in_by %} by {{ connection.signed_in_by }}{% endif %}, as {{ "an administrator: Steward can install updates" if connection.can_install else "a user who isn't an administrator: Steward shows status only" }}.</p>
+{% endif %}
+<form method="post" action="/plugins/homeassistant/connection/sign-in">{{ csrf_field() }}
+  <section class="card form-section" aria-labelledby="ha-sign-in">
+    <h2 id="ha-sign-in" class="section-title">Sign in with Home Assistant</h2>
+    <div class="field">
+      <label class="field-label" for="ha-url">Address</label>
+      <input class="input mono" id="ha-url" name="url" required maxlength="500" value="{{ url or '' }}" placeholder="http://homeassistant.local:8123" autocomplete="off">
+      <span class="field-hint">Both your browser and Steward need to reach it at this address.</span>
+    </div>
+    <label style="display:flex;gap:0.5rem;align-items:center;cursor:pointer;margin-bottom:0.9rem;">
+      <input type="checkbox" name="verify_tls" value="on"{% if verify_tls %} checked{% endif %}> Check its HTTPS certificate
+    </label>
+    <p class="tone-warn"><strong>Steward stores a credential with the same access as the user who signs in.</strong> Home Assistant has no narrower permission, so an administrator's credential can do anything that administrator can, including controlling every device. Steward only uses it to read status, take backups and install updates.</p>
+    <p class="field-hint"><strong>We recommend a service account:</strong> a Home Assistant user made just for Steward (Settings → People → Add person). What Steward does then appears under its own name in Home Assistant's logs, so it's easy to tell apart from what people do.</p>
+    <p class="field-hint" style="margin-bottom:0;">Installing updates needs an administrator. With any other user, Steward shows status only.</p>
+  </section>
+
+  <div class="form-actions">
+    <button type="submit" class="btn">{{ "Sign in again" if connection else "Continue to Home Assistant" }}</button>
+    <a href="/plugins/homeassistant/" class="btn btn-ghost">Cancel</a>
+  </div>
+</form>
+
+{% if connection %}
+<section class="card danger-zone" aria-labelledby="ha-remove">
+  <div>
+    <h2 id="ha-remove" class="section-title">Remove this connection</h2>
+    <p class="field-hint">Steward stops reading Home Assistant, forgets its credential and ends the sign-in there.</p>
+  </div>
+  <form method="post" action="/plugins/homeassistant/connection/delete"
+        data-confirm="Remove the Home Assistant connection? Steward stops reading it and forgets its credential.">{{ csrf_field() }}
+    <button class="btn btn-danger" type="submit">Remove connection</button>
+  </form>
+</section>
+{% endif %}
+</div>
+{% endblock %}

plugins/homeassistant/templates/homeassistant/index.html

@@ -0,0 +1,150 @@
+{# plugins/homeassistant/templates/homeassistant/index.html — whether Home
+   Assistant answers, which updates are waiting with Install beside each, how its
+   backups stand, and the updates Steward has run. Refreshes itself while one is
+   under way. #}
+{% extends "base.html" %}
+{% from "_macros.html" import page_header, status_badge, stat_tile, empty_state %}
+{% block title %}Home Assistant — Steward{% endblock %}
+{% block content %}
+{% set admin = session.user_role == 'admin' %}
+{% call page_header("Home Assistant", connection.url if connection else none) %}
+  {% if admin and connection %}<a class="btn btn-sm btn-ghost" href="/plugins/homeassistant/connection">Connection</a>{% endif %}
+{% endcall %}
+
+{% if not connection %}
+{% if error %}<p class="field-error" role="alert">{{ error }}</p>{% endif %}
+<div class="card">{{ empty_state("Not connected. Sign in to a Home Assistant instance to see its waiting updates and backups, and to install updates after a confirmed backup.",
+                                 "Connect" if admin else none, "/plugins/homeassistant/connection") }}</div>
+{% else %}
+{% if error %}<p class="field-error" role="alert">{{ error }}</p>{% endif %}
+<div id="ha-body"{% if under_way %} hx-get="/plugins/homeassistant/" hx-select="#ha-body" hx-trigger="every 15s" hx-swap="outerHTML"{% endif %}>
+<p class="page-verdict">
+  {% if under_way %}{{ status_badge('pending', 'Updating') }}
+    {{ under_way.title }}: {{ {'backing_up': "taking a backup first", 'rebooting': "rebooting into " ~ under_way.to_version}.get(under_way.state, "installing " ~ under_way.to_version) }}. Home Assistant may stop answering for a while.
+  {% elif reach == 'up' %}{{ status_badge('up', 'Answering') }}
+    {% if installable %}{{ installable|length }} update{{ "s" if installable|length != 1 }} waiting.{% else %}Everything Steward can update is up to date.{% endif %}
+    Read {{ connection.last_polled_at|ago }}.
+  {% elif reach == 'down' %}{{ status_badge('down', 'Not answering') }} <span class="tone-crit">{{ says }}</span>
+    {% if connection.last_ok_at %}Last answered {{ connection.last_ok_at|ago }}; what's below is from then.{% endif %}
+    {% if admin and connection.refused %}<form method="post" action="/plugins/homeassistant/connection/retry" style="display:inline;">{{ csrf_field() }}<button class="btn btn-sm btn-ghost" type="submit">Try again</button></form>{% endif %}
+  {% else %}{{ status_badge('pending', 'Not read yet') }} {{ says }}{% endif %}
+</p>
+
+{% if not connection.can_install %}
+<p class="tone-warn">The Home Assistant user who signed in isn't an administrator, so updates can't be installed from here.
+  {% if admin %}<a href="/plugins/homeassistant/connection">Sign in again</a> as an administrator to install them.{% endif %}</p>
+{% endif %}
+
+<div class="stat-strip">
+  {{ stat_tile("System problems", problems|length if health.repairs_read else "—", state="degraded" if problems else none) }}
+  {{ stat_tile("Storage", ("%s of %s GB (%.0f%%)"|format(health.disk_used_gb, health.disk_total_gb, health.disk_used_pct)) if health.disk_used_pct is number else "—") }}
+  {{ stat_tile("Updates waiting", installable|length if connection.last_ok_at else "—") }}
+  {{ stat_tile("Last good backup", (backup_at|ago) if backup_at else "—", state="degraded" if backup_tone == 'warn' else none) }}
+  {{ stat_tile("Next backup", (backup_next|ago) if backup_next else "—") }}
+</div>
+<p class="{{ 'tone-warn' if backup_tone == 'warn' else 'field-hint' }}">{{ backup_says }}</p>
+
+<section class="card" aria-labelledby="ha-system">
+  <h2 id="ha-system" class="section-title">System</h2>
+  {% if problems %}
+  <table class="table">
+    <thead><tr><th scope="col">Problem</th><th scope="col">Severity</th></tr></thead>
+    <tbody>
+    {% for p in problems %}<tr><td>{{ p.title }}</td><td>{{ status_badge('down' if p.severity in ('critical', 'error') else 'degraded', p.severity|capitalize) }}</td></tr>{% endfor %}
+    </tbody>
+  </table>
+  <p class="field-hint">Fix these in Home Assistant: Settings → System → Repairs.</p>
+  {% elif health.repairs_read %}
+  <p class="field-hint">No repairs open, and nothing wrong reported.</p>
+  {% elif connection.last_ok_at %}
+  <p class="tone-warn">Home Assistant wouldn't share its repairs with this user. {% if admin %}<a href="/plugins/homeassistant/connection">Sign in again</a> as an administrator to see them.{% endif %}</p>
+  {% endif %}
+  {% if health.repairs_read and not health.supervised %}<p class="field-hint">No Supervisor answered, so disk space and add-ons aren't checked here.</p>{% endif %}
+  {% if health.error %}<p class="tone-warn" style="margin-bottom:0;">This is from an earlier read. {{ health.error }}</p>{% endif %}
+</section>
+
+<section class="card" aria-labelledby="ha-updates">
+  <h2 id="ha-updates" class="section-title">Updates</h2>
+  {% if installable %}
+  <table class="table">
+    <thead><tr><th scope="col">What</th><th scope="col">Now</th><th scope="col">New</th>{% if admin %}<th scope="col"></th>{% endif %}</tr></thead>
+    <tbody>
+    {% for u in installable %}
+      <tr>
+        <td>{{ u.title }}<div class="cell-sub">{{ u.kind_word }}{% if u.auto_update %}, updates itself{% endif %}</div></td>
+        <td class="mono">{{ u.installed or "—" }}</td>
+        <td class="mono">{% if u.release_url %}<a href="{{ u.release_url }}" rel="noopener noreferrer" target="_blank">{{ u.latest or "—" }}</a>{% else %}{{ u.latest or "—" }}{% endif %}</td>
+        {% if admin %}<td style="text-align:right;">
+          {% if connection.can_install and not under_way and not connection.refused and not u.in_progress %}
+          <form method="post" action="/plugins/homeassistant/install"
+                data-confirm="Install {{ u.title }} {{ u.latest }}? Steward takes a backup first and won't install without it. {{ 'Home Assistant restarts.' if u.kind == 'core' else ('It takes effect once you reboot the machine, which Steward leaves to you.' if u.kind == 'os' else 'The add-on restarts.') }}">{{ csrf_field() }}
+            <input type="hidden" name="entity_id" value="{{ u.entity_id }}">
+            <button class="btn btn-sm" type="submit">Install</button>
+          </form>
+          {% endif %}
+        </td>{% endif %}
+      </tr>
+    {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p class="field-hint" style="margin:0;">Nothing waiting.</p>
+  {% endif %}
+  {% if current %}<p class="field-hint">Up to date: {% for u in current %}{{ u.title }} {{ u.installed }}{{ ", " if not loop.last }}{% endfor %}.</p>{% endif %}
+  {% if others %}<p class="field-hint" style="margin-bottom:0;">Waiting, for Home Assistant to install: {% for u in others %}{{ u.title }}{{ ", " if not loop.last }}{% endfor %}. Steward installs Core, OS and add-on updates only.</p>{% endif %}
+</section>
+
+<section class="card" aria-labelledby="ha-schedule">
+  <h2 id="ha-schedule" class="section-title">Scheduled updates</h2>
+  {% if plan and plan.enabled %}
+  <p class="field-hint">{{ plan_when }}. Next: {{ plan_next|ago }}. Add-ons first, then Core, then the OS and its reboot, each after a confirmed backup. A failure ends the run and the rest wait.</p>
+  {% else %}
+  <p class="field-hint">Off. Updates are only installed when someone chooses Install.</p>
+  {% endif %}
+  {% if plan and plan.stopped %}<p class="tone-warn">The last run stopped early. {{ plan.stopped }}</p>{% endif %}
+  {% if admin %}
+  <p class="tone-warn">A Core update restarts Home Assistant and an OS update reboots the machine, with nobody there to watch. An update can also change how automations behave.</p>
+  <form method="post" action="/plugins/homeassistant/schedule" style="display:flex;flex-wrap:wrap;gap:0.75rem;align-items:flex-end;">{{ csrf_field() }}
+    <label style="display:flex;gap:0.5rem;align-items:center;cursor:pointer;margin-bottom:0.4rem;">
+      <input type="checkbox" name="enabled" value="on"{% if plan and plan.enabled %} checked{% endif %}> Install on a schedule
+    </label>
+    <div class="field" style="margin-bottom:0;">
+      <label class="field-label" for="ha-day">Day</label>
+      <select class="input" id="ha-day" name="weekday">{% for d in weekdays %}<option value="{{ loop.index0 }}"{% if loop.index0 == (plan.weekday if plan else 5) %} selected{% endif %}>{{ d }}</option>{% endfor %}</select>
+    </div>
+    <div class="field" style="margin-bottom:0;">
+      <label class="field-label" for="ha-time">Time</label>
+      <input class="input" id="ha-time" type="time" name="at_time" required value="{{ plan.at_time if plan else '05:00' }}">
+    </div>
+    <div class="field" style="margin-bottom:0;">
+      <label class="field-label" for="ha-zone">Time zone</label>
+      <input class="input" id="ha-zone" name="time_zone" required value="{{ plan.time_zone if plan else default_zone }}">
+    </div>
+    <button class="btn btn-sm" type="submit">Save</button>
+  </form>
+  {% endif %}
+</section>
+
+{% if runs %}
+<section class="card" aria-labelledby="ha-runs">
+  <h2 id="ha-runs" class="section-title">Updates run from Steward</h2>
+  <table class="table">
+    <thead><tr><th scope="col">What</th><th scope="col">Versions</th><th scope="col">Outcome</th><th scope="col">Started</th></tr></thead>
+    <tbody>
+    {% for r in runs %}
+      <tr>
+        <td>{{ r.title }}<div class="cell-sub">by {{ r.started_by or "Steward" }}</div></td>
+        <td class="mono">{{ r.from_version or "—" }} → {{ r.to_version or "—" }}</td>
+        <td>{% if r.state == 'done' %}{{ status_badge('up', 'Done') }}
+          {% elif r.state == 'failed' %}{{ status_badge('down', 'Failed') }}<div class="cell-sub">{{ r.message }}</div>
+          {% else %}{{ status_badge('pending', {'backing_up': 'Taking a backup', 'rebooting': 'Rebooting'}.get(r.state, 'Installing')) }}{% endif %}</td>
+        <td>{{ r.started_at|ago }}</td>
+      </tr>
+    {% endfor %}
+    </tbody>
+  </table>
+</section>
+{% endif %}
+</div>
+{% endif %}
+{% endblock %}

plugins/homeassistant/timeline.py

@@ -0,0 +1,34 @@
+# plugins/homeassistant/timeline.py
+"""What Home Assistant maintenance puts on the timeline: each update once it ends."""
+from __future__ import annotations
+
+from steward.timeline import catalog
+from steward.timeline.record import record_or_update
+
+TYPES = (
+    ("homeassistant.updated", "Home Assistant updated", "automation", "service", True,
+     "Steward installed a Home Assistant update after a confirmed backup."),
+    ("homeassistant.update_failed", "Home Assistant update failed", "automation", "service", True,
+     "A Home Assistant update Steward was asked to install didn't finish."),
+)
+
+
+def register() -> None:
+    for t in TYPES:
+        catalog.register_type(*t)
+    from .models import HomeAssistantUpdate
+    catalog.register_ref("homeassistant_update", HomeAssistantUpdate)
+
+
+async def update_entry(session, run) -> None:
+    done = run.state == "done"
+    versions = f"{run.from_version} to {run.to_version}" if run.to_version else "a new version"
+    summary = (f"{run.title} updated from {versions}" if done
+               else f"{run.title} update failed: {run.message}")
+    await record_or_update(
+        session, ("homeassistant_update", run.id),
+        type="homeassistant.updated" if done else "homeassistant.update_failed",
+        at=run.started_at, until=run.finished_at, summary=summary, subject=run.title,
+        detail={"entity_id": run.entity_id, "from": run.from_version, "to": run.to_version,
+                "url": "/plugins/homeassistant/"},
+        actor=run.started_by or "Steward")

plugins/homeassistant/updates.py

@@ -0,0 +1,259 @@
+# plugins/homeassistant/updates.py
+"""Installing one update: take a backup, confirm it landed, install, and follow
+it to its end. Its row holds where it has got to, so Steward restarting, or Home
+Assistant restarting as an update does, only delays the next step."""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+
+from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError
+
+from . import client
+from .models import HomeAssistantConnection, HomeAssistantUpdate
+from .service import BACKUP_EVENT, LAST_BACKUP, is_newer, parse_moment
+from .timeline import update_entry
+
+logger = logging.getLogger(__name__)
+
+UNDER_WAY = ("backing_up", "installing", "rebooting")
+BACKUP_DEADLINE_S = 30 * 60
+REBOOT_DEADLINE_S = 20 * 60
+# An OS update downloads and writes a whole system image, which is slow on a Pi. Home
+# Assistant reports the new version as soon as that's written, before the reboot that
+# makes it run, and raises a repair asking for that reboot. The schedule's own OS
+# updates make that reboot; one started by a person leaves it to them.
+INSTALL_DEADLINE_S = {"addon": 20 * 60, "core": 30 * 60, "os": 45 * 60}
+
+
+class CannotStart(Exception):
+    """Why an update wasn't started, said to the person."""
+
+
+async def _mark_refused(app, connection_id: str, said: str) -> None:
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            row = await session.get(HomeAssistantConnection, connection_id)
+            if row is not None:
+                row.refused, row.last_error = True, said
+
+
+async def under_way(session) -> HomeAssistantUpdate | None:
+    return (await session.execute(select(HomeAssistantUpdate).where(
+        HomeAssistantUpdate.state.in_(UNDER_WAY)))).scalars().first()
+
+
+async def start_update(app, entity_id: str, username: str, transport=None, *, reboot_after: bool = False) -> HomeAssistantUpdate:
+    """Start installing `entity_id`, an explicit action that may contact Home
+    Assistant. Raises CannotStart, with nothing started, unless the update is
+    still waiting and the signed-in user may install it. `reboot_after` reboots
+    the machine once it's written, and it's done when the new version is running."""
+    async with app.db_sessionmaker() as session:
+        connection = (await session.execute(select(HomeAssistantConnection))).scalars().first()
+        if connection is None:
+            raise CannotStart("Home Assistant isn't connected")
+        if connection.refused:
+            raise CannotStart("Home Assistant refused Steward's sign-in")
+        if await under_way(session) is not None:
+            raise CannotStart("another update is under way")
+        if not connection.can_install:
+            raise CannotStart("the Home Assistant user who signed in isn't an administrator")
+    url, verify = connection.url, connection.verify_tls
+    known = next((u for u in connection.updates or [] if u["entity_id"] == entity_id), None)
+    if known is None or not known["kind"]:
+        raise CannotStart("that isn't a Core, OS or add-on update")
+    try:
+        token = await client.access_token(connection, transport)
+        current = await client.read_state(url, entity_id, token, verify, transport)
+        backup = await client.read_state(url, LAST_BACKUP, token, verify, transport)
+        event = await client.read_state(url, BACKUP_EVENT, token, verify, transport)
+    except client.Refused as exc:
+        await _mark_refused(app, connection.id, str(exc))
+        raise CannotStart(str(exc).rstrip(".")) from None
+    except client.Unreachable as exc:
+        raise CannotStart(str(exc).rstrip(".")) from None
+    if backup is None:
+        raise CannotStart("Home Assistant has no automatic backups to confirm one with")
+    if current is None or current.get("state") != "on":
+        raise CannotStart("that update is no longer waiting")
+    attrs = current.get("attributes") or {}
+    run = HomeAssistantUpdate(
+        entity_id=entity_id, title=known["title"], kind=known["kind"],
+        from_version=attrs.get("installed_version"), to_version=attrs.get("latest_version"),
+        started_by=username, backup_before=backup.get("state"),
+        backup_event_before=(event or {}).get("state"), reboot_after=reboot_after)
+    async with app.db_sessionmaker() as session:
+        try:
+            async with session.begin():
+                session.add(run)
+                await session.flush()
+                run_id = run.id
+        except IntegrityError:
+            raise CannotStart("another update is under way") from None
+    try:
+        await client.call_action(url, "backup", "create_automatic", {}, token, verify, transport)
+    except client.Refused as exc:
+        await _mark_refused(app, connection.id, str(exc))
+        await _finish(app, run_id, "failed", f"The backup couldn't be started: {exc} Nothing was installed.")
+        raise CannotStart(str(exc).rstrip(".")) from None
+    except (client.Unreachable, client.Rejected) as exc:
+        await _finish(app, run_id, "failed", f"The backup couldn't be started: {exc} Nothing was installed.")
+        raise CannotStart(str(exc).rstrip(".")) from None
+    async with app.db_sessionmaker() as session:
+        return await session.get(HomeAssistantUpdate, run_id)
+
+
+async def _finish(app, run_id: str, state: str, message: str | None) -> None:
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            run = await session.get(HomeAssistantUpdate, run_id)
+            run.state, run.message, run.finished_at = state, message, datetime.now(timezone.utc)
+            # The page should show the new versions now, not at the next poll.
+            connection = (await session.execute(select(HomeAssistantConnection))).scalars().first()
+            if connection is not None:
+                connection.last_polled_at = None
+            try:
+                async with session.begin_nested():
+                    await update_entry(session, run)
+            except Exception:
+                logger.exception("Home Assistant update %s: its timeline entry could not be written", run_id)
+
+
+async def advance(app, transport=None, connect=None) -> None:
+    """Move the update under way one step on, if there is one."""
+    async with app.db_sessionmaker() as session:
+        run = await under_way(session)
+        connection = (await session.execute(select(HomeAssistantConnection))).scalars().first()
+    if run is None:
+        return
+    if connection is None:
+        await _finish(app, run.id, "failed", "The Home Assistant connection was removed.")
+        return
+    try:
+        if run.state == "backing_up":
+            await _advance_backup(app, run, connection, transport)
+        elif run.state == "installing":
+            await _advance_install(app, run, connection, transport)
+        else:
+            await _advance_reboot(app, run, connection, transport, connect)
+    except client.Refused as exc:
+        await _mark_refused(app, connection.id, str(exc))
+        await _finish(app, run.id, "failed", str(exc))
+    except client.Rejected as exc:
+        await _finish(app, run.id, "failed", f"Home Assistant wouldn't install it: {exc}")
+    except client.Unreachable as exc:
+        # Expected while Home Assistant restarts: the next tick asks again.
+        logger.info("Home Assistant update %s: %s", run.id, exc)
+        await _past_deadline(app, run)
+
+
+def _elapsed(since: datetime) -> float:
+    return (datetime.now(timezone.utc) - since).total_seconds()
+
+
+async def _past_deadline(app, run) -> bool:
+    if run.state == "backing_up" and _elapsed(run.started_at) > BACKUP_DEADLINE_S:
+        await _finish(app, run.id, "failed",
+                      f"No new backup within {BACKUP_DEADLINE_S // 60} minutes. Nothing was installed.")
+        return True
+    if run.state == "installing":
+        limit = INSTALL_DEADLINE_S.get(run.kind, INSTALL_DEADLINE_S["os"])
+        if _elapsed(run.install_sent_at or run.backup_at or run.started_at) > limit:
+            await _finish(app, run.id, "failed",
+                          f"Still not on {run.to_version} after {limit // 60} minutes. Look in Home Assistant.")
+            return True
+    if run.state == "rebooting" and _elapsed(run.reboot_sent_at or run.install_sent_at or run.started_at) > REBOOT_DEADLINE_S:
+        await _finish(app, run.id, "failed", f"{run.to_version} was installed, but isn't running "
+                      f"{REBOOT_DEADLINE_S // 60} minutes after the reboot. Look in Home Assistant.")
+        return True
+    return False
+
+
+async def _advance_backup(app, run, connection, transport) -> None:
+    token = await client.access_token(connection, transport)
+    url, verify = connection.url, connection.verify_tls
+    backup = await client.read_state(url, LAST_BACKUP, token, verify, transport)
+    value = (backup or {}).get("state")
+    if is_newer(value, run.backup_before):
+        async with app.db_sessionmaker() as session:
+            async with session.begin():
+                row = await session.get(HomeAssistantUpdate, run.id)
+                row.state, row.backup_at = "installing", parse_moment(value)
+        return
+    event = await client.read_state(url, BACKUP_EVENT, token, verify, transport)
+    if event and event.get("state") != run.backup_event_before \
+            and (event.get("attributes") or {}).get("event_type") == "failed":
+        reason = (event.get("attributes") or {}).get("failed_reason")
+        await _finish(app, run.id, "failed",
+                      f"The backup failed{f' ({reason})' if reason else ''}. Nothing was installed.")
+        return
+    await _past_deadline(app, run)
+
+
+async def _mark_sent(app, run_id: str, at: datetime | None) -> None:
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            row = await session.get(HomeAssistantUpdate, run_id)
+            row.install_sent_at = at
+
+
+async def _advance_install(app, run, connection, transport) -> None:
+    token = await client.access_token(connection, transport)
+    url, verify = connection.url, connection.verify_tls
+    if run.install_sent_at is None:
+        # Marked first: Steward stopping here leaves an install that runs out its
+        # deadline, where asking twice would fail one that is going fine.
+        await _mark_sent(app, run.id, datetime.now(timezone.utc))
+        try:
+            await client.call_action(url, "update", "install", {"entity_id": run.entity_id}, token, verify, transport)
+        except client.Unreachable:
+            await _mark_sent(app, run.id, None)
+            raise
+        return
+    current = await client.read_state(url, run.entity_id, token, verify, transport)
+    attrs = (current or {}).get("attributes") or {}
+    installed = attrs.get("installed_version")
+    if installed and (installed == run.to_version
+                      or (current.get("state") == "off" and installed != run.from_version)):
+        if run.reboot_after:
+            async with app.db_sessionmaker() as session:
+                async with session.begin():
+                    row = await session.get(HomeAssistantUpdate, run.id)
+                    row.state = "rebooting"
+            return
+        await _finish(app, run.id, "done", None)
+        return
+    await _past_deadline(app, run)
+
+
+async def _mark_reboot_sent(app, run_id: str, at: datetime | None) -> None:
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            row = await session.get(HomeAssistantUpdate, run_id)
+            row.reboot_sent_at = at
+
+
+async def _advance_reboot(app, run, connection, transport, connect) -> None:
+    """Reboot once, and wait for the new version to be what's running. The version
+    is judged against the old one, so a machine that fell back to it fails at the
+    deadline, as does one that never comes back."""
+    token = await client.access_token(connection, transport)
+    url, verify = connection.url, connection.verify_tls
+    if run.reboot_sent_at is None:
+        await _mark_reboot_sent(app, run.id, datetime.now(timezone.utc))
+        try:
+            await client.call_action(url, "hassio", "host_reboot", {}, token, verify, transport)
+        except client.Unreachable:
+            await _mark_reboot_sent(app, run.id, None)
+            await _past_deadline(app, run)
+            return
+        except client.Rejected as exc:
+            await _finish(app, run.id, "failed", f"{run.to_version} was installed, but Home Assistant "
+                          f"wouldn't reboot: {exc} It needs a reboot to take effect.")
+        return
+    found = await client.read_os(url, token, verify, connect)
+    if found and found.get("version") not in (None, run.from_version) and not found.get("version_pending"):
+        await _finish(app, run.id, "done", None)
+        return
+    await _past_deadline(app, run)

plugins/host_agent/__init__.py

@@ -11,11 +11,51 @@ _app: "Quart | None" = None
 def setup(app: "Quart") -> None:
     global _app
     _app = app
+    # A host's CPU, memory and load per core learn their normal (#5742), each
+    # with its floor from Settings → Thresholds. Parts don't: a host's CPU has
+    # a normal, each of its cores doesn't. Disks stay on their fixed cutoffs:
+    # a nearly full disk is a failure whatever its history (System 182).
+    from steward.core.norms import floor_setting, learn_normal
+    learn_normal("host_agent", "cpu_pct", floor_setting(app, "host_cpu_pct"), parts=False)
+    learn_normal("host_agent", "mem_used_pct", floor_setting(app, "host_mem_pct"), parts=False)
+    learn_normal("host_agent", "load_pct", floor_setting(app, "host_load_pct"), parts=False)
+    # "Agent offline" speaks for everything the agent carries: while it is
+    # firing, the host's other rules going silent isn't emailed (#5721).
+    from steward.core.alerts import agent_fed, agent_heartbeat
+    from steward.hosts.service import agent_stale_after_seconds
+    agent_heartbeat("host_agent", "seconds_since_seen", lambda: agent_stale_after_seconds(app.config))
+    agent_fed("host_agent", lambda resource: resource.split(":", 1)[0])
+    # temp_c_max stands for one of many sensors: its alert says which.
+    from steward.core.alerts import names_behind
+    from .routes import name_hottest_sensor
+    names_behind("host_agent", "temp_c_max", name_hottest_sensor)
+    # get_host over MCP marks the sensors the host's temperature leaves out.
+    from steward.mcp.tools.hosts import judge_sensors
+    from .routes import split_temps
+    judge_sensors(lambda host, temps: {k: why for k, (_c, why) in split_temps(host, temps)[1].items()})
+    # A reboot the agent announced is silence with a reason, for a while (#5756).
+    from datetime import datetime, timezone
+    from steward.core.alerts import agents_away
+    from .downtime import hosts_away, restart_grace_seconds
+    agents_away(lambda session: hosts_away(session, datetime.now(timezone.utc),
+                                           restart_grace_seconds(app.config)))
+    from steward.core.cleanup import prune_by_age
+    from .models import HostDowntime
+    prune_by_age(HostDowntime.started_at)
+    # Maintenance runs and the stretches a host was away go on the timeline.
+    from .timeline import register as register_timeline
+    register_timeline()
+
+
+def get_audit_entities() -> dict:
+    """Audit rows about a maintenance plan name it and link to the plans page."""
+    from .models import HostMaintenancePlan
+    return {"maintenance_plan": (HostMaintenancePlan, "/plugins/host_agent/maintenance")}
 
 
 def get_scheduled_tasks() -> list:
-    from .scheduler import make_task
+    from .scheduler import make_maintenance_task, make_sensor_task, make_task
-    return [make_task(_app)]
+    return [make_task(_app), make_maintenance_task(_app), make_sensor_task(_app)]
 
 
 def get_blueprint():
@@ -33,12 +73,77 @@ async def mcp_overview(db) -> list[dict]:
     return await findings(db)
 
 
+async def timeline_backfill(session, since) -> list:
+    from .timeline import backfill
+    return await backfill(session, since)
+
+
 def get_alert_metrics() -> dict[str, list[str]]:
     """Host metrics ingest records for alert rules (resource = the host name;
     disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
-    comes from the scheduler, not the agent: "agent offline" is > 180."""
+    comes from the scheduler, not the agent: "agent offline" is > 180.
+    clock_off_secs is how far the host's clock is from Steward's, either way
+    (agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a %
+    of the host's cores. inodes_used_pct is per mount (agent ≥ 1.13.0), and
+    disk_used_pct_worst is the fullest mount by space or inodes.
+    mounts_stalled counts the mounts that didn't answer the agent in time
+    (agent ≥ 1.14.0): "an NFS share stopped" is > 0. agent_rss_bytes and
+    agent_cpu_pct are the agent's own memory and CPU (agent ≥ 1.15.0): a
+    healthy agent sits in tens of MB and a few %. updates_pending and
+    updates_security are what the package manager has waiting, and
+    reboot_required is 1 when the host needs a reboot (agent ≥ 1.16.0):
+    "security updates waiting" is updates_security > 0. maintenance_failed
+    comes from the scheduler (#5768): 1 when the host's latest maintenance job
+    failed or timed out, 0 once one succeeds: "maintenance failed" is > 0.
+    temp_c_max is the host's hottest real thermometer, in °C (#5787); sensors
+    with nothing wired to them are left out (#5759, #5783): "running hot" is
+    > 85. temp_c is one sensor, resource "<host>:temp:<label>". net_rx_bps /
+    net_tx_bps and disk_read_bps / disk_write_bps are bytes per second, for
+    the whole host ("<host>") or one interface ("<host>:net:<iface>") or disk
+    ("<host>:diskio:<device>").
+
+    NFS delay (agent ≥ 1.22.0, #5996), per NFS mount on "<host>:<mount>",
+    over the requests it finished since the last sample: nfs_exe_ms is the
+    average time a program waited on one (queueing included), nfs_rtt_ms the
+    server's round trip, nfs_read_ms / nfs_write_ms the same for reads and
+    writes alone, nfs_ops_per_sec how many, and nfs_retrans / nfs_timeouts the
+    retransmissions and major timeouts among them; nfs_tx_bps / nfs_rx_bps
+    (agent ≥ 1.25.0, #6115) are the bytes a second it sent and received, its
+    share of the host's network traffic. nfs_exe_max_ms is the
+    host's slowest mount, for a covering rule: "NFS slow" is > 1000 for 3
+    checks. A healthy share answers in a few ms.
+
+    Drive health (agent ≥ 1.20.0, #5789), read hourly with smartctl by the
+    root helper and judged in drives.py, per drive on "<host>:drive:<device>":
+    drive_failing is 1 when the drive fails its own self-assessment, raises
+    an NVMe critical warning or runs out of spare blocks; drive_life_used_pct
+    is how much of its rated life it has used; drive_bad_sectors counts its
+    media errors or reallocated / pending / uncorrectable sectors;
+    drive_temp_c is its own temperature. Per host, for covering rules:
+    drives_failing (how many) and drive_life_used_max_pct (the most worn):
+    "drive failing" is drives_failing > 0, "drive wearing out" is
+    drive_life_used_max_pct > 90.
+
+    Recorded but not offered (NOT_ALERTABLE): disk_used_bytes and
+    disk_total_bytes, where disk_used_pct is the figure to watch (and docker
+    uses disk_total_bytes for something else), and mem_cached_bytes /
+    mem_buffers_bytes, parts of what mem_available_bytes already counts."""
+    from .drives import DRIVE_METRICS
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
-        "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m",
+        "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
-        "uptime_secs", "seconds_since_seen",
+        "load_pct",
+        "uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled",
+        "agent_rss_bytes", "agent_cpu_pct",
+        "updates_pending", "updates_security", "reboot_required", "maintenance_failed",
+        "temp_c_max", "temp_c", "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
+        "nfs_exe_max_ms", "nfs_exe_ms", "nfs_rtt_ms", "nfs_read_ms", "nfs_write_ms",
+        "nfs_ops_per_sec", "nfs_retrans", "nfs_timeouts", "nfs_tx_bps", "nfs_rx_bps",
+        *DRIVE_METRICS,
     ]}
+
+
+# Metrics host_agent records that no rule is offered, each for the reason
+# get_alert_metrics gives; tests/core/test_metric_catalog.py holds every other
+# recorded name to the catalog.
+NOT_ALERTABLE = frozenset({"disk_used_bytes", "disk_total_bytes", "mem_cached_bytes", "mem_buffers_bytes"})

plugins/host_agent/cadence.py

@@ -0,0 +1,281 @@
+# plugins/host_agent/cadence.py
+"""A group's maintenance cadence (#5768): every week, its hosts updated one at
+a time, each rebooting only when its update needs it (or as the plan says).
+
+A plan names an inventory group, a weekday and a time in its time zone, and a
+window. At that time a run starts with the group's hosts in a fixed order. Each
+host gets a maintenance job (maintenance.py); the next host starts only when
+the last one is back in service: its agent posting, its swarm node ready and
+active, every replicated service at its replicas. A job that fails, or a host
+that doesn't come back into service, stops the run, and the host's
+maintenance_failed figure goes to 1 for an alert rule to catch. No host is
+started after the window; the run then stops and says which hosts it didn't
+reach. A run that drained swarm nodes ends by spreading the swarm's services
+back out (rebalance.py), unless its plan says not to.
+
+The decisions are model-free here (plans and runs are anything with the
+attributes); the database side is below them, and the plugin's minute tick
+drives it. Everything is on the rows, so a restart of Steward, or the reboot of
+the host it runs on, picks up where it was.
+"""
+from __future__ import annotations
+
+import json
+from datetime import datetime, timedelta
+
+from steward.core.weekly import WEEKDAYS, describe, fired_slot, is_due, last_slot, next_slot, parse_at  # noqa: F401
+
+from . import maintenance, rebalance, timeline
+
+# How long a host may take to be back in service after its job finished.
+SETTLE_LIMIT = timedelta(minutes=30)
+# An agent heard from this recently is posting.
+POSTING_WITHIN = timedelta(minutes=3)
+RUN_STATES = ("running", "done", "failed", "stopped")
+
+
+def back_in_service(*, last_seen_at, node, services, job_finished_at, now: datetime,
+                    expect_active: bool = True) -> str | None:
+    """None when the host is back in service, else why not yet. `node` is the
+    freshest swarm-node row for the job's node (None outside a swarm, or when
+    the job has no node); `services` the freshest row per swarm service. Only
+    rows reported after the job finished count, so a stale "all fine" from
+    before the reboot never passes. `expect_active` is False for a node the
+    job didn't drain (the operator keeps it drained): it need only be ready."""
+    if last_seen_at is None or now - last_seen_at > POSTING_WITHIN:
+        return "its agent isn't posting yet"
+    if node is None:
+        return None
+    if node.updated_at < job_finished_at:
+        return "no swarm report since it came back"
+    if node.status != "ready" or (expect_active and node.availability != "active"):
+        return f"its swarm node is {node.status}/{node.availability}"
+    fresh = [s for s in services if s.updated_at >= job_finished_at]
+    if not fresh:
+        return "no swarm report since it came back"
+    short = sorted(s.service_name for s in fresh
+                   if s.mode == "replicated" and s.running < s.desired)
+    if short:
+        return "services below their replicas: " + ", ".join(short[:5])
+    return None
+
+
+def next_move(run, plan, job, back: str | None, now: datetime) -> tuple[str, str | None]:
+    """What a running run does now: ("start", None) the current host's job;
+    ("wait", None); ("next", None) on to the next host; ("settling", why);
+    ("fail", why); ("stop", why) the window ended; ("done", None)."""
+    hosts = json.loads(run.host_ids)
+    if run.position >= len(hosts):
+        return "done", None
+    if job is None:
+        if now >= run.started_at + timedelta(minutes=plan.window_minutes):
+            left = len(hosts) - run.position
+            return "stop", f"the window ended with {left} host{'s' if left != 1 else ''} not started"
+        return "start", None
+    if job.state in maintenance.ACTIVE:
+        return "wait", None
+    if job.state != "done":
+        return "fail", job.detail or maintenance.LABELS[job.state]
+    if back is None:
+        return "next", None
+    if run.settle_since is not None and now - run.settle_since > SETTLE_LIMIT:
+        return "fail", f"{maintenance.NOT_BACK} {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}"
+    return "settling", back
+
+
+# ── applied to the database: the minute tick, Run now, the page ──────────────
+
+async def group_hosts(session, group_id: str) -> list:
+    """The group's hosts (through their linked Ansible targets), by name."""
+    from sqlalchemy import select
+    from steward.models.ansible_inventory import AnsibleTarget, ansible_target_groups
+    from steward.models.hosts import Host
+    rows = (await session.execute(
+        select(Host).join(AnsibleTarget, AnsibleTarget.host_id == Host.id)
+        .join(ansible_target_groups, ansible_target_groups.c.target_id == AnsibleTarget.id)
+        .where(ansible_target_groups.c.group_id == group_id))).scalars().all()
+    return sorted({h.id: h for h in rows}.values(), key=lambda h: h.name.lower())
+
+
+async def start_run(session, plan, now: datetime, manual: bool = False):
+    """A run of the plan with the group's hosts in order. A host whose agent
+    has no maintenance helper is passed over, and the run says so."""
+    from sqlalchemy import select
+    from .models import HostAgentRegistration, HostMaintenanceRun
+    hosts = await group_hosts(session, plan.group_id)
+    regs = {r.host_id: r for r in (await session.execute(select(HostAgentRegistration).where(
+        HostAgentRegistration.host_id.in_([h.id for h in hosts])))).scalars()} if hosts else {}
+    order, skipped = [], []
+    for h in hosts:
+        reg = regs.get(h.id)
+        if reg is None:
+            skipped.append({"host_id": h.id, "reason": "no agent"})
+        elif not reg.maint_helper:
+            skipped.append({"host_id": h.id, "reason": "no maintenance helper: run Update agent"})
+        else:
+            order.append(h.id)
+    run = HostMaintenanceRun(plan_id=plan.id, state="running", host_ids=json.dumps(order),
+                             position=0, skipped=json.dumps(skipped), manual=manual,
+                             started_at=now)
+    session.add(run)
+    await session.flush()
+    return run
+
+
+async def _back(session, host, job, reg, now: datetime) -> str | None:
+    from sqlalchemy import select
+    node = None
+    services: list = []
+    if job.node_id:
+        models = maintenance.docker_models()
+        if models is not None:
+            DockerSwarmNode, DockerSwarmService = models.DockerSwarmNode, models.DockerSwarmService
+            node = (await session.execute(select(DockerSwarmNode).where(
+                DockerSwarmNode.node_id == job.node_id)
+                .order_by(DockerSwarmNode.updated_at.desc()).limit(1))).scalar_one_or_none()
+            freshest: dict = {}
+            for s in (await session.execute(select(DockerSwarmService))).scalars():
+                if s.service_name not in freshest or s.updated_at > freshest[s.service_name].updated_at:
+                    freshest[s.service_name] = s
+            services = list(freshest.values())
+    return back_in_service(last_seen_at=reg.last_seen_at if reg else None, node=node,
+                           services=services, job_finished_at=job.finished_at or now, now=now,
+                           expect_active=bool(job.drain_host_id))
+
+
+async def advance(session, run, plan, now: datetime) -> str:
+    """Move one running run on as far as it can go now; returns its state."""
+    from sqlalchemy import select
+    from steward.models.hosts import Host
+    from .models import HostAgentRegistration, HostMaintenanceJob
+    for _ in range(64):                             # a host per pass; bounded
+        hosts = json.loads(run.host_ids)
+        host_id = hosts[run.position] if run.position < len(hosts) else None
+        job = host = reg = None
+        if host_id is not None:
+            host = await session.get(Host, host_id)
+            reg = (await session.execute(select(HostAgentRegistration).where(
+                HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
+            job = (await session.execute(select(HostMaintenanceJob).where(
+                HostMaintenanceJob.run_id == run.id, HostMaintenanceJob.host_id == host_id)
+                .order_by(HostMaintenanceJob.created_at.desc()).limit(1))).scalar_one_or_none()
+        back = None
+        if job is not None and job.state == "done":
+            back = await _back(session, host, job, reg, now)
+        move, why = next_move(run, plan, job, back, now)
+        if move == "start":
+            if host is None:
+                _skip(run, host_id, "the host is gone")
+                continue
+            try:
+                await maintenance.create_job(session, host, reg, plan.created_by,
+                                             plan.reboot_policy, now, run_id=run.id)
+            except maintenance.CannotStart as exc:
+                _skip(run, host_id, str(exc))
+                continue
+            return run.state
+        if move == "next":
+            run.position += 1
+            run.settle_since = None
+            continue
+        if move == "settling":
+            run.settle_since = run.settle_since or now
+            return run.state
+        if move == "fail":
+            if job is not None and job.state == "done":
+                maintenance._finish(job, "failed", now, why)     # the host never came back into service
+                await timeline.write(session, f"maintenance job {job.id}",
+                                     timeline.job_entry(session, job, host.name if host else None))
+            name = host.name if host else host_id
+            _end(run, "failed", f"{name}: {why}", now=now)
+        elif move == "stop":
+            _end(run, "stopped", why, now=now)
+        elif move == "done":
+            if await rebalance.begin(session, run, plan, now) or \
+                    (run.rebalance or {}).get("state") == "handed":
+                return run.state            # a manager is spreading services back out
+            _end(run, *rebalance.outcome(run), now=now)
+        return run.state
+    return run.state
+
+
+def _skip(run, host_id: str, reason: str) -> None:
+    skipped = json.loads(run.skipped)
+    skipped.append({"host_id": host_id, "reason": reason})
+    run.skipped = json.dumps(skipped)
+    hosts = json.loads(run.host_ids)
+    hosts.pop(run.position)
+    run.host_ids = json.dumps(hosts)
+
+
+def _end(run, state: str, detail: str | None = None, *, now: datetime) -> None:
+    run.state, run.finished_at, run.settle_since = state, now, None
+    if detail:
+        run.detail = detail[:maintenance.DETAIL_MAX]
+
+
+async def tick(session, now: datetime) -> list[str]:
+    """Start due plans and move running runs on; returns lines worth logging."""
+    from sqlalchemy import select
+    from .models import HostMaintenancePlan, HostMaintenanceRun
+    said = []
+    running = {r.plan_id: r for r in (await session.execute(select(HostMaintenanceRun).where(
+        HostMaintenanceRun.state == "running"))).scalars()}
+    for plan in (await session.execute(select(HostMaintenancePlan))).scalars():
+        slot = is_due(plan, now)
+        if slot is None:
+            continue
+        plan.last_slot_at = slot
+        if plan.id in running:
+            said.append(f"plan {plan.name!r}: the {slot:%Y-%m-%d %H:%M}Z slot came with a run still going")
+            continue
+        running[plan.id] = await start_run(session, plan, now)
+        said.append(f"plan {plan.name!r}: run started")
+    for run in running.values():
+        plan = await session.get(HostMaintenancePlan, run.plan_id)
+        if plan is None:
+            continue
+        if rebalance.check_deadline(run, now):
+            await rebalance.finish(session, run, now)
+        before = run.state
+        after = await advance(session, run, plan, now)
+        if after != before:
+            said.append(f"plan {plan.name!r}: run {after}" + (f" ({run.detail})" if run.detail else ""))
+    return said
+
+
+async def failed_hosts(session, now: datetime) -> dict[str, float]:
+    """{host name: 1 or 0} for every host with an agent: whether its latest
+    finished maintenance job failed or timed out, for the maintenance_failed
+    figure an alert rule watches. A host with no job yet reads 0, so a rule
+    on it reports rather than sitting at "no data". A later job that
+    succeeds puts it back to 0, and so does a host that failed only for not
+    coming back into service in time, once it is back (#6105). That return
+    is kept on the job (came_back_at), so a later wobble in the host's
+    services doesn't flag it again (#6175)."""
+    from sqlalchemy import select
+    from steward.models.hosts import Host
+    from .models import HostAgentRegistration, HostMaintenanceJob
+    agents = (await session.execute(
+        select(Host.name).join(HostAgentRegistration, HostAgentRegistration.host_id == Host.id))).scalars()
+    out = {name: 0.0 for name in agents}
+    rows = (await session.execute(
+        select(Host, HostMaintenanceJob)
+        .join(Host, Host.id == HostMaintenanceJob.host_id)
+        .where(HostMaintenanceJob.state.in_(maintenance.FINISHED),
+               HostMaintenanceJob.finished_at >= now - timedelta(days=30)))).all()
+    latest: dict = {}
+    for host, job in rows:
+        if host.name not in latest or job.finished_at > latest[host.name][1].finished_at:
+            latest[host.name] = (host, job)
+    for name, (host, job) in latest.items():
+        if (job.state == "failed" and job.came_back_at is None
+                and (job.detail or "").startswith(maintenance.NOT_BACK)):
+            reg = (await session.execute(select(HostAgentRegistration).where(
+                HostAgentRegistration.host_id == host.id))).scalar_one_or_none()
+            if await _back(session, host, job, reg, now) is None:
+                job.came_back_at = now
+                await timeline.write(session, f"maintenance job {job.id}",
+                                     timeline.job_entry(session, job, host.name))
+        out[name] = 1.0 if maintenance.still_failed(job) else 0.0
+    return out

plugins/host_agent/downtime.py

@@ -0,0 +1,172 @@
+# plugins/host_agent/downtime.py
+"""What a host's agent says happened to it while it was away (#5756).
+
+An agent ≥ 1.14.0 tells Steward when it stops (and whether its host is going
+down with it), when it starts (and when its host booted), and how long it
+couldn't reach Steward. Each becomes a HostDowntime row, so a reboot reads as
+a reboot with its length, and "agent offline" is left for silence nobody
+explained. A post can be retried, so every event is applied idempotently.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import select
+
+from .models import HostDowntime
+
+# A stop the agent announced: the silence after it is explained, for a while.
+ANNOUNCED = ("reboot", "agent_stop")
+
+LABELS = {
+    "reboot": "Rebooted",
+    "agent_stop": "Agent stopped",
+    "unclean": "Restarted without shutting down",
+    "lost_contact": "Lost contact",
+}
+
+
+def restart_grace_seconds(config) -> int:
+    """How long an announced stop explains the agent's silence (Settings →
+    Plugins → host_agent, default 10 minutes): past it, agent offline fires."""
+    return int(((config.get("PLUGINS") or {}).get("host_agent") or {})
+               .get("reboot_grace_seconds", 600))
+
+
+def _at(value) -> datetime | None:
+    if not isinstance(value, str):
+        return None
+    try:
+        got = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return got if got.tzinfo else got.replace(tzinfo=timezone.utc)
+
+
+def _when(event: dict) -> datetime | None:
+    return _at(event.get("at") or event.get("from"))
+
+
+async def record_events(session, host_id: str, events, last_seen_at: datetime | None) -> int:
+    """Apply one post's events to the host's downtime, oldest first; returns
+    how many rows were added or changed. `last_seen_at` is when the agent was
+    last heard before this post: a boot after it that nobody announced is a
+    restart without a shutdown. Each row added or changed goes on the
+    timeline, kept up to date as it ends."""
+    if not isinstance(events, list):
+        return 0
+    timed = sorted(((t, e) for e in events if isinstance(e, dict)
+                    for t in [_when(e)] if t is not None), key=lambda te: te[0])
+    changed = []
+    for at, event in timed:
+        kind = event.get("kind")
+        row = None
+        if kind == "stopping":
+            row = await _stopping(session, host_id, at, event.get("host_shutdown"))
+        elif kind == "started":
+            row = await _started(session, host_id, at, _at(event.get("boot_at")), last_seen_at)
+        elif kind == "lost_contact":
+            row = await _lost_contact(session, host_id, at, _at(event.get("to")))
+        if row is not None and row not in changed:
+            changed.append(row)
+    if changed:
+        from . import timeline
+        await session.flush()
+        for row in changed:
+            await timeline.write(session, f"host downtime {row.id}", timeline.downtime_entry(session, row))
+    return len(changed)
+
+
+async def _stopping(session, host_id: str, at: datetime, host_shutdown) -> HostDowntime | None:
+    seen = (await session.execute(select(HostDowntime.id).where(
+        HostDowntime.host_id == host_id, HostDowntime.kind.in_(ANNOUNCED),
+        HostDowntime.started_at == at))).first()
+    if seen:
+        return None
+    row = HostDowntime(host_id=host_id, kind="reboot" if host_shutdown else "agent_stop", started_at=at)
+    session.add(row)
+    return row
+
+
+async def _started(session, host_id: str, at: datetime, boot_at: datetime | None,
+                   last_seen_at: datetime | None) -> HostDowntime | None:
+    stop = (await session.execute(select(HostDowntime).where(
+        HostDowntime.host_id == host_id, HostDowntime.kind.in_(ANNOUNCED),
+        HostDowntime.ended_at.is_(None), HostDowntime.started_at <= at)
+        .order_by(HostDowntime.started_at.desc()).limit(1))).scalar_one_or_none()
+    if stop is not None:
+        stop.ended_at = at
+        # The boot time settles what the stop was: the host came back from a
+        # boot after it, or it stayed up while only the agent restarted.
+        if boot_at is not None:
+            stop.kind = "reboot" if boot_at > stop.started_at else "agent_stop"
+        return stop
+    if boot_at is None or last_seen_at is None or boot_at <= last_seen_at:
+        return None
+    seen = (await session.execute(select(HostDowntime.id).where(
+        HostDowntime.host_id == host_id, HostDowntime.kind == "unclean",
+        HostDowntime.ended_at == at))).first()
+    if seen:
+        return None
+    row = HostDowntime(host_id=host_id, kind="unclean", started_at=last_seen_at, ended_at=at)
+    session.add(row)
+    return row
+
+
+async def _lost_contact(session, host_id: str, since: datetime,
+                        until: datetime | None) -> HostDowntime | None:
+    if until is None or until <= since:
+        return None
+    row = (await session.execute(select(HostDowntime).where(
+        HostDowntime.host_id == host_id, HostDowntime.kind == "lost_contact",
+        HostDowntime.started_at == since))).scalar_one_or_none()
+    if row is None:
+        row = HostDowntime(host_id=host_id, kind="lost_contact", started_at=since, ended_at=until)
+        session.add(row)
+        return row
+    if row.ended_at is None or until > row.ended_at:
+        row.ended_at = until
+        return row
+    return None
+
+
+async def announced_stops(session, now: datetime, grace_seconds: int) -> dict[str, datetime]:
+    """{host id: when the silence its agent announced stops being explained}
+    for every announced stop still going and inside its grace."""
+    rows = (await session.execute(select(HostDowntime.host_id, HostDowntime.started_at).where(
+        HostDowntime.kind.in_(ANNOUNCED), HostDowntime.ended_at.is_(None),
+        HostDowntime.started_at > now - timedelta(seconds=grace_seconds)))).all()
+    out: dict[str, datetime] = {}
+    for host_id, started in rows:
+        until = started + timedelta(seconds=grace_seconds)
+        out[host_id] = max(out.get(host_id, until), until)
+    return out
+
+
+async def hosts_away(session, now: datetime, grace_seconds: int) -> set[str]:
+    """Names of the hosts in announced_stops: their silence has a reason."""
+    from steward.models.hosts import Host
+    ids = list(await announced_stops(session, now, grace_seconds))
+    if not ids:
+        return set()
+    return set((await session.execute(select(Host.name).where(Host.id.in_(ids)))).scalars())
+
+
+def describe(row: HostDowntime, now: datetime) -> dict:
+    """One downtime row as the host page shows it."""
+    label = LABELS.get(row.kind, row.kind)
+    if row.kind == "agent_stop" and row.ended_at is not None:
+        label = "Agent restarted"
+    end = row.ended_at or now
+    return {"kind": row.kind, "label": label, "started_at": row.started_at,
+            "ended_at": row.ended_at, "ongoing": row.ended_at is None,
+            "seconds": max(0.0, (end - row.started_at).total_seconds())}
+
+
+async def recent(session, host_id: str, now: datetime, days: int = 7, limit: int = 6) -> list[dict]:
+    """The host's downtime in the last `days`, newest first."""
+    rows = (await session.execute(select(HostDowntime).where(
+        HostDowntime.host_id == host_id,
+        HostDowntime.started_at > now - timedelta(days=days))
+        .order_by(HostDowntime.started_at.desc()).limit(limit))).scalars().all()
+    return [describe(r, now) for r in rows]

plugins/host_agent/drives.py

@@ -0,0 +1,116 @@
+"""Each drive's health, judged from what the root helper read with smartctl
+(#5789). The helper reports figures, not verdicts, so how a drive is judged
+can change here without an agent update.
+
+A drive is:
+- "failing" when its own self-assessment fails (SMART overall-health), an NVMe
+  drive raises a critical warning, or its spare blocks fall under threshold;
+- "worn" when it has used WORN_PCT of its rated life;
+- "watch" when it has bad sectors or media errors but otherwise passes: old
+  disks carry a few for years, so a count alone isn't a failure;
+- "ok", or "unavailable" when it has no SMART to read (a virtual disk, a
+  bridge that hides it), or "unreadable" when smartctl couldn't read it.
+
+Metrics, per drive on "<host>:drive:<device>": drive_failing (1/0),
+drive_life_used_pct, drive_bad_sectors, drive_temp_c. Per host, for covering
+rules: drives_failing (how many) and drive_life_used_max_pct.
+"""
+from __future__ import annotations
+
+WORN_PCT = 90
+DRIVE_METRICS = ("drive_failing", "drive_life_used_pct", "drive_bad_sectors", "drive_temp_c",
+                 "drives_failing", "drive_life_used_max_pct")
+# What a drive's own figures count as bad sectors or media errors, whichever
+# its kind reports: NVMe media errors, ATA reallocated / pending / offline
+# uncorrectable sectors, the SCSI grown defect list.
+_BAD = ("media_errors", "reallocated", "pending", "offline_uncorrectable", "grown_defects")
+_ORDER = {"failing": 0, "unreadable": 1, "worn": 2, "watch": 3, "ok": 4, "unavailable": 5}
+
+
+def _num(v):
+    return v if isinstance(v, (int, float)) and not isinstance(v, bool) else None
+
+
+def bad_sectors(d: dict) -> int | None:
+    counts = [_num(d.get(k)) for k in _BAD]
+    counts = [c for c in counts if c is not None]
+    return int(sum(counts)) if counts else None
+
+
+def failing_reasons(d: dict) -> list[str]:
+    why = []
+    if d.get("passed") is False:
+        why.append("its SMART self-assessment failed")
+    if _num(d.get("critical_warning")):
+        why.append("critical warning %#x" % int(d["critical_warning"]))
+    spare, floor = _num(d.get("spare_pct")), _num(d.get("spare_threshold_pct"))
+    if spare is not None and floor is not None and spare < floor:
+        why.append(f"spare blocks {spare:.0f}% under the {floor:.0f}% threshold")
+    return why
+
+
+def judge(d: dict) -> dict:
+    """One drive as the host page shows it: its figures plus `state` and `why`."""
+    out = dict(d)
+    if d.get("error"):
+        out.update(state="unreadable", why=str(d["error"]))
+    elif d.get("available") is False:
+        out.update(state="unavailable", why="no SMART data (a virtual disk, or SMART is off)")
+    elif failing_reasons(d):
+        out.update(state="failing", why="; ".join(failing_reasons(d)))
+    elif (_num(d.get("life_used_pct")) or 0) >= WORN_PCT:
+        out.update(state="worn", why=f"{d['life_used_pct']:.0f}% of its rated life used")
+    elif bad_sectors(d):
+        out.update(state="watch", why=f"{bad_sectors(d)} bad sectors or media errors")
+    else:
+        out.update(state="ok", why="")
+    out["bad_sectors"] = bad_sectors(d)
+    return out
+
+
+def drives_of(report) -> list[dict]:
+    """The report's drives, judged, what needs attention first. A name seen
+    twice gets an ordinal, so two drives never share one series: agent
+    1.20.0 named every disk behind a RAID controller "0" (#5806)."""
+    drives = report.get("drives") if isinstance(report, dict) else None
+    judged, seen = [], {}
+    for d in drives or []:
+        if not isinstance(d, dict) or not d.get("device"):
+            continue
+        d = judge(d)
+        name = str(d["device"])
+        seen[name] = seen.get(name, 0) + 1
+        if seen[name] > 1:
+            d["device"] = f"{name}#{seen[name]}"
+        judged.append(d)
+    return sorted(judged, key=lambda d: (_ORDER[d["state"]], str(d["device"])))
+
+
+def metric_rows(report, host_name: str) -> list[tuple[str, str, float]]:
+    """(metric, resource, value) for every figure a drive has, and the host's
+    two roll-ups once any drive was read."""
+    rows: list[tuple[str, str, float]] = []
+    failing = 0
+    life_max: float | None = None
+    read = 0
+    for d in drives_of(report):
+        if d["state"] in ("unreadable", "unavailable"):
+            continue
+        read += 1
+        res = f"{host_name}:drive:{d['device']}"
+        is_failing = d["state"] == "failing"
+        failing += is_failing
+        rows.append(("drive_failing", res, 1.0 if is_failing else 0.0))
+        life = _num(d.get("life_used_pct"))
+        if life is not None:
+            rows.append(("drive_life_used_pct", res, float(life)))
+            life_max = life if life_max is None else max(life_max, life)
+        if d["bad_sectors"] is not None:
+            rows.append(("drive_bad_sectors", res, float(d["bad_sectors"])))
+        if _num(d.get("temp_c")) is not None:
+            rows.append(("drive_temp_c", res, float(d["temp_c"])))
+    if read:
+        rows.append(("drives_failing", host_name, float(failing)))
+        if life_max is not None:
+            rows.append(("drive_life_used_max_pct", host_name, float(life_max)))
+    return rows

plugins/host_agent/files/install-deps.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+# Steward host agent: the packages the agent's helper uses, installed with
+# whichever package manager the host has (#5789). Served by Steward at
+# /plugins/host_agent/files/install-deps.sh and run, as root, by install.sh
+# and the host_agent playbooks: one copy of the list.
+#
+# smartmontools gives the helper smartctl, which reads each drive's health.
+# On pacman hosts, pacman-contrib gives the agent checkupdates, the one safe
+# way to see pending updates without root (#5803).
+# A package that can't be installed is reported, never fatal: the agent runs
+# without it and the host page says what is missing.
+#
+# Recommended / weak dependencies are skipped: on Debian, smartmontools
+# recommends a mail server.
+
+
+have() { command -v "$1" >/dev/null 2>&1; }
+
+install_with() {
+    if have apt-get; then
+        export DEBIAN_FRONTEND=noninteractive
+        # A stale package list can't find the package: refresh once and retry.
+        apt-get install -y --no-install-recommends "$@" \
+            || { apt-get update && apt-get install -y --no-install-recommends "$@"; }
+    elif have dnf; then
+        dnf -y install --setopt=install_weak_deps=False "$@"
+    elif have yum; then
+        yum -y install "$@"
+    elif have zypper; then
+        zypper --non-interactive install --no-recommends "$@"
+    elif have pacman; then
+        pacman -S --needed --noconfirm "$@"
+    elif have apk; then
+        apk add --no-cache "$@"
+    else
+        echo "no package manager this script knows (apt, dnf, yum, zypper, pacman, apk)"
+        return 1
+    fi
+}
+
+missing=""
+have smartctl || missing="$missing smartmontools"
+if have pacman && ! have checkupdates; then
+    missing="$missing pacman-contrib"
+fi
+[ -n "$missing" ] || { echo "dependencies present"; exit 0; }
+
+# shellcheck disable=SC2086  # one word per package
+if install_with $missing; then
+    echo "installed:$missing"
+else
+    echo "WARNING: could not install$missing; until it is, drive health reads \"smartctl not installed\" (smartmontools) and the update check reads \"checkupdates isn't installed\" (pacman-contrib)"
+fi
+exit 0

plugins/host_agent/files/maint.py

@@ -0,0 +1,646 @@
+#!/usr/bin/env python3
+"""Steward maintenance helper (#5766): runs as root, started by steward-maint.path
+when the host agent leaves a request, and does only what ACTIONS lists.
+
+The agent (an unprivileged user under NoNewPrivileges) can't update or reboot
+its host itself. It writes `<id>.json` = {"id", "action", ["node"]} into
+REQUESTS, the one directory it may write. This helper reads each request
+without following links, refuses anything outside the list, deletes it, does
+the action, and writes the outcome to RESULTS (root's own directory, which the
+agent reads back and reports to Steward). A stolen agent token can therefore
+make its host update, reboot or drain, or on a swarm manager restart named
+services where the swarm puts them, and nothing else.
+
+One action only reads: `drives` (#5789) asks smartctl for each disk's health,
+which needs root, and writes a summary to DRIVES, never a result: it is not a
+maintenance job. `check` (#6098) changes nothing but the package lists: it
+says what an update would upgrade, whether Docker is among it, and which
+containers run outside the swarm.
+
+`rebalance` spreads swarm services back out after a maintenance run left them
+on one node. Steward names the services, having checked that the emptier node
+holds each one's image; the helper force-updates them one at a time, spec
+unchanged and no image pulled, so the swarm places each on its emptiest node.
+A service a constraint or a preference places is left alone.
+
+Stdlib only; served by Steward at /plugins/host_agent/files/maint.py.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import time
+from datetime import datetime, timezone
+
+REQUESTS = "/var/lib/steward-agent/requests"
+RESULTS = "/var/lib/steward-maint/results"
+DRIVES = "/var/lib/steward-maint/drives.json"
+ACTIONS = ("check", "update", "reboot", "drain", "undrain", "drives", "rebalance")
+REQUEST_MAX_BYTES = 4096
+RESULTS_KEPT_SECONDS = 7 * 86400
+UPDATE_DEADLINE_SECONDS = 3000
+COMMAND_DEADLINE_SECONDS = 600
+OUTPUT_KEPT_CHARS = 2000
+# A listing of what an update would change is read whole, not just its tail.
+LISTING_KEPT_CHARS = 1_000_000
+_ID = re.compile(r"^[A-Za-z0-9-]{1,64}$")
+_NODE = re.compile(r"^[a-z0-9]{25}$")
+# A swarm service's name as Docker allows it, never a flag.
+_SERVICE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$")
+SERVICES_MAX = 48
+MOVE_DEADLINE_SECONDS = 300
+# A rebalance starts no move after this long.
+REBALANCE_SECONDS = 2400
+
+
+def now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def read_request(path: str):
+    """The request at `path` as a dict, or None when it is not a small regular
+    file holding a valid request. Never follows a link the agent planted."""
+    try:
+        fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
+    except OSError:
+        return None
+    try:
+        st = os.fstat(fd)
+        if not (st.st_mode & 0o170000 == 0o100000) or st.st_size > REQUEST_MAX_BYTES:
+            return None
+        raw = os.read(fd, REQUEST_MAX_BYTES + 1)
+    finally:
+        os.close(fd)
+    try:
+        req = json.loads(raw.decode("utf-8"))
+    except (UnicodeDecodeError, ValueError):
+        return None
+    return req if valid(req) else None
+
+
+def valid(req) -> bool:
+    """A request names a known action, has a plain id, a node id only in the
+    swarm's own form, and, for a rebalance alone, a list of service names."""
+    if not isinstance(req, dict) or req.get("action") not in ACTIONS:
+        return False
+    if not isinstance(req.get("id"), str) or not _ID.match(req["id"]):
+        return False
+    services = req.get("services")
+    if req["action"] == "rebalance":
+        if not (isinstance(services, list) and 0 < len(services) <= SERVICES_MAX
+                and all(isinstance(s, str) and _SERVICE.match(s) for s in services)):
+            return False
+    elif services is not None:
+        return False
+    node = req.get("node")
+    return node is None or (isinstance(node, str) and bool(_NODE.match(node)))
+
+
+def run(cmd: list, deadline: int = COMMAND_DEADLINE_SECONDS, env: dict | None = None,
+        ok_codes: tuple = (0,), keep: int | None = OUTPUT_KEPT_CHARS):
+    """(ok, output tail) of a command, bounded by `deadline`. `ok_codes` are
+    the exit codes that mean it worked (zypper says more than 0); `keep` is
+    how much of the output's end is kept, None for all of it."""
+    try:
+        done = subprocess.run(cmd, capture_output=True, text=True, timeout=deadline,
+                              env=dict(os.environ, LANG="C", LC_ALL="C", **(env or {})))
+    except subprocess.TimeoutExpired:
+        return False, f"{cmd[0]} didn't finish in {deadline} s"
+    except OSError as exc:
+        return False, str(exc)
+    out = (done.stdout or "") + (done.stderr or "")
+    return done.returncode in ok_codes, out if keep is None else out[-keep:]
+
+
+REBOOT_MARKER = "/var/run/reboot-required"                 # Debian and Ubuntu
+KERNEL_MODULES = "/usr/lib/modules"
+# zypper's own exit codes for "done, and": 102 a reboot is needed, 103 zypper
+# updated itself and would run again (its next run picks up the rest).
+ZYPPER_OK = (0, 102, 103)
+
+
+def reboot_required() -> bool | None:
+    """Whether the host needs a reboot after an update, asked the way its
+    package manager says it; None when it can't say."""
+    if os.path.exists(REBOOT_MARKER):
+        return True
+    if shutil.which("apt-get"):
+        return False
+    try:
+        if shutil.which("needs-restarting"):                   # dnf, yum
+            return subprocess.run(["needs-restarting", "-r"], capture_output=True,
+                                  timeout=60).returncode == 1
+        if shutil.which("zypper"):
+            code = subprocess.run(["zypper", "needs-rebooting"], capture_output=True,
+                                  timeout=60).returncode
+            return {0: False, 102: True}.get(code)
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    if shutil.which("pacman"):
+        # An upgraded kernel removes the running one's modules.
+        return not os.path.isdir(os.path.join(KERNEL_MODULES, os.uname().release))
+    return None
+
+
+def upgraded_count(output: str) -> int | None:
+    """Packages an update changed, from the manager's summary line."""
+    m = re.search(r"(\d+) upgraded, (\d+) newly installed", output)       # apt
+    if m:
+        return int(m.group(1)) + int(m.group(2))
+    m = re.search(r"(\d+) packages? to upgrade(?:, (\d+) new)?", output)  # zypper
+    if m:
+        return int(m.group(1)) + int(m.group(2) or 0)
+    m = re.search(r"^Packages \((\d+)\)", output, re.MULTILINE)           # pacman
+    if m:
+        return int(m.group(1))
+    counts = re.findall(r"^\s*(?:Upgrade|Install)\s+(\d+) Packages?", output, re.MULTILINE)  # dnf, yum
+    return sum(int(c) for c in counts) if counts else None
+
+
+# The same as the agent's: this helper runs as root on its own and imports
+# nothing of the agent's, so it keeps its own copy.
+def _os_release_id(path: str = "/etc/os-release") -> str:
+    try:
+        with open(path) as fh:
+            for line in fh:
+                if line.startswith("ID="):
+                    return line.split("=", 1)[1].strip().strip('"')
+    except OSError:
+        pass
+    return ""
+
+
+def update_steps(manager: str) -> list:
+    """The commands an update runs on a host with `manager`, in order: each
+    (what it is, command, deadline, env, ok exit codes). The last one is the upgrade
+    whose output is counted. Existing config files are kept, and needrestart
+    only lists, leaving service restarts to a reboot. A package's own install
+    script can still restart its service: Docker's does, which is why the job
+    checks first and drains a swarm node before such an update (#6098)."""
+    long = UPDATE_DEADLINE_SECONDS
+    if manager == "apt":
+        env = {"DEBIAN_FRONTEND": "noninteractive", "NEEDRESTART_MODE": "l"}
+        return [("apt-get update", ["apt-get", "update"], COMMAND_DEADLINE_SECONDS, env, (0,)),
+                ("apt-get dist-upgrade", ["apt-get", "-y", "-o", "Dpkg::Options::=--force-confdef",
+                                          "-o", "Dpkg::Options::=--force-confold", "dist-upgrade"],
+                 long, env, (0,))]
+    if manager == "dnf":
+        return [("dnf upgrade", ["dnf", "-y", "upgrade", "--refresh"], long, None, (0,))]
+    if manager == "yum":
+        return [("yum update", ["yum", "-y", "update"], long, None, (0,))]
+    if manager == "zypper":
+        # Tumbleweed is a rolling release: it upgrades with dup, not update.
+        upgrade = "dup" if _os_release_id() == "opensuse-tumbleweed" else "update"
+        return [("zypper refresh", ["zypper", "--non-interactive", "refresh"],
+                 COMMAND_DEADLINE_SECONDS, None, ZYPPER_OK),
+                ("zypper " + upgrade, ["zypper", "--non-interactive", upgrade, "--auto-agree-with-licenses"],
+                 long, None, ZYPPER_OK)]
+    if manager == "pacman":
+        # Packages from the AUR aren't updated: no AUR helper runs as root.
+        return [("pacman -Syu", ["pacman", "-Syu", "--noconfirm"], long, None, (0,))]
+    return []
+
+
+# Which manager a host has; dnf before yum, since dnf hosts often keep a yum
+# that is dnf underneath.
+UPDATE_MANAGERS = (("apt", "apt-get"), ("dnf", "dnf"), ("yum", "yum"), ("zypper", "zypper"),
+                   ("pacman", "pacman"))
+
+
+def host_manager() -> str | None:
+    for manager, binary in UPDATE_MANAGERS:
+        if shutil.which(binary):
+            return manager
+    return None
+
+
+def do_update() -> dict:
+    """Upgrade every package with whichever package manager the host has."""
+    manager = host_manager()
+    if manager is None:
+        return {"ok": False, "detail": "no package manager this helper knows "
+                                       "(apt, dnf, yum, zypper, pacman) on this host"}
+    out = ""
+    for label, cmd, deadline, env, ok_codes in update_steps(manager):
+        # The whole output: every manager prints its count before the long
+        # log of what it then does, so a tail loses it on a large update.
+        ok, out = run(cmd, deadline=deadline, env=env, ok_codes=ok_codes, keep=None)
+        if not ok:
+            return {"ok": False, "detail": f"{label} failed: " + out[-OUTPUT_KEPT_CHARS:]}
+    changed = upgraded_count(out)
+    if manager == "apt":
+        run(["apt-get", "-y", "autoremove"], env={"DEBIAN_FRONTEND": "noninteractive"})
+    return {"ok": True, "packages": changed, "reboot_required": reboot_required(),
+            "detail": out[-500:]}
+
+
+# ── what an update would change, and what runs outside the swarm (#6098) ─────
+# Docker's own packages restart the daemon from their install scripts, whatever
+# needrestart is told, and every container on the host stops with it. So the
+# job asks first: a swarm node is drained before an update that upgrades
+# Docker, not after it, and the containers running outside the swarm are
+# noted, to see each of them running again once the job is over.
+
+DOCKER_PACKAGES = re.compile(
+    r"^(docker|docker-ce|docker\.io|docker-engine|moby-engine|containerd|containerd\.io)$")
+CONTAINERS_MAX = 40
+CONTAINER_NAME_MAX = 64
+
+
+def pending_steps(manager: str) -> list:
+    """The commands that list what an update would upgrade, changing nothing
+    but the package lists: each (what it is, command, ok exit codes). The
+    last one's output names the packages."""
+    if manager == "apt":
+        return [("apt-get update", ["apt-get", "update"], (0,)),
+                ("apt-get -s dist-upgrade", ["apt-get", "-s", "dist-upgrade"], (0,))]
+    if manager == "dnf":
+        # check-update exits 100 when there is something to update.
+        return [("dnf check-update", ["dnf", "-q", "check-update", "--refresh"], (0, 100))]
+    if manager == "yum":
+        return [("yum check-update", ["yum", "-q", "check-update"], (0, 100))]
+    if manager == "zypper":
+        if _os_release_id() == "opensuse-tumbleweed":
+            listing = ("zypper dup --dry-run", ["zypper", "--non-interactive", "dup", "--dry-run",
+                                                "--auto-agree-with-licenses"])
+        else:
+            listing = ("zypper list-updates", ["zypper", "--non-interactive", "list-updates"])
+        return [("zypper refresh", ["zypper", "--non-interactive", "refresh"], ZYPPER_OK),
+                (*listing, ZYPPER_OK)]
+    if manager == "pacman":
+        # -Sy refreshes the lists only; the update's -Syu follows it. -Qu exits
+        # 1 when there is nothing to upgrade.
+        return [("pacman -Sy", ["pacman", "-Sy"], (0,)), ("pacman -Qu", ["pacman", "-Qu"], (0, 1))]
+    return []
+
+
+def pending_names(manager: str, output: str) -> list:
+    """The packages a listing from pending_steps says would be upgraded."""
+    if manager == "apt":
+        return re.findall(r"^Inst (\S+)", output, re.MULTILINE)
+    if manager in ("dnf", "yum"):
+        # "docker-ce.x86_64   3:27.1.1-1.el9   docker-ce-stable"
+        return [parts[0].rsplit(".", 1)[0] for parts in (line.split() for line in output.splitlines())
+                if len(parts) == 3 and "." in parts[0]]
+    if manager == "zypper":
+        names, column, listing = [], None, False
+        for line in output.splitlines():
+            cells = [c.strip() for c in line.split("|")]
+            if len(cells) > 2 and column is None and "Name" in cells:
+                column = cells.index("Name")                   # list-updates' table
+            elif column is not None and len(cells) > column and cells[0] == "v":
+                names.append(cells[column])
+            elif re.search(r"going to be (upgraded|installed)", line):
+                listing = True                                 # dup --dry-run's lists
+            elif listing and line.startswith(" ") and line.strip():
+                names.extend(line.split())
+            else:
+                listing = False
+        return names
+    if manager == "pacman":
+        return [line.split()[0] for line in output.splitlines() if line.strip()]
+    return []
+
+
+def standalone_containers() -> list:
+    """The running containers that aren't swarm tasks, each with its restart
+    policy, by name; at most CONTAINERS_MAX, so the result stays small."""
+    if not shutil.which("docker"):
+        return []
+    ok, out = run(["docker", "ps", "-q", "--no-trunc"], deadline=30, keep=LISTING_KEPT_CHARS)
+    ids = out.split() if ok else []
+    if not ids:
+        return []
+    fmt = '{{.Name}}|{{.HostConfig.RestartPolicy.Name}}|{{index .Config.Labels "com.docker.swarm.task.id"}}'
+    ok, out = run(["docker", "inspect", "--format", fmt, *ids], deadline=60, keep=LISTING_KEPT_CHARS)
+    if not ok:
+        return []
+    found = []
+    for line in out.splitlines():
+        parts = line.split("|")
+        if len(parts) != 3 or parts[2].strip():
+            continue                                    # a swarm task: the drain moves it
+        found.append({"name": parts[0].lstrip("/")[:CONTAINER_NAME_MAX],
+                      "restart": parts[1].strip() or "no"})
+    return sorted(found, key=lambda c: c["name"])[:CONTAINERS_MAX]
+
+
+def do_check() -> dict:
+    """What an update would upgrade, whether Docker is among it, and the
+    containers running outside the swarm. Only the package lists change."""
+    manager = host_manager()
+    if manager is None:
+        return {"ok": False, "detail": "no package manager this helper knows "
+                                       "(apt, dnf, yum, zypper, pacman) on this host"}
+    env = {"DEBIAN_FRONTEND": "noninteractive"} if manager == "apt" else None
+    out = ""
+    for label, cmd, ok_codes in pending_steps(manager):
+        ok, out = run(cmd, env=env, ok_codes=ok_codes, keep=LISTING_KEPT_CHARS)
+        if not ok:
+            return {"ok": False, "detail": f"{label} failed: " + out[-500:]}
+    names = pending_names(manager, out)
+    docker = sorted({n for n in names if DOCKER_PACKAGES.match(n)})
+    return {"ok": True, "pending": len(names), "docker_update": bool(docker),
+            "docker_packages": docker[:10], "containers": standalone_containers()}
+
+
+def own_node() -> str | None:
+    ok, out = run(["docker", "info", "--format", "{{.Swarm.NodeID}}"], deadline=30)
+    node = out.strip() if ok else ""
+    return node if _NODE.match(node) else None
+
+
+def do_node(availability: str, node: str | None) -> dict:
+    """Set a swarm node's availability (drain before a reboot, active after).
+    Only a manager can; a worker's helper says so and Steward asks a manager."""
+    if not shutil.which("docker"):
+        return {"ok": False, "detail": "no docker on this host"}
+    node = node or own_node()
+    if node is None:
+        return {"ok": False, "detail": "this host is not in a swarm"}
+    ok, out = run(["docker", "node", "update", "--availability", availability, node], deadline=60)
+    return {"ok": ok, "node": node, "detail": out.strip()[-500:]}
+
+
+def placed_by_rule(name: str) -> bool | None:
+    """Whether a constraint or a preference decides where the service runs
+    (a force-update would leave it where it is); None when it can't be read."""
+    ok, out = run(["docker", "service", "inspect", "--format",
+                   "{{json .Spec.TaskTemplate.Placement}}", name], deadline=30)
+    if not ok:
+        return None
+    try:
+        placement = json.loads(out.strip() or "null") or {}
+    except ValueError:
+        return None
+    return bool(placement.get("Constraints") or placement.get("Preferences"))
+
+
+def do_rebalance(services: list, clock=time.monotonic) -> dict:
+    """Force-update each named service in turn, waiting for it to settle, so
+    the swarm starts it again on its emptiest node. Fails when a move didn't
+    settle; what was left and why is in `left`."""
+    if not shutil.which("docker"):
+        return {"ok": False, "detail": "no docker on this host"}
+    began = clock()
+    moved, left, failed = [], [], []
+    for name in services:
+        if clock() - began > REBALANCE_SECONDS:
+            left.append({"service": name, "reason": "out of time"})
+            continue
+        rule = placed_by_rule(name)
+        if rule is None:
+            left.append({"service": name, "reason": "not found on this manager"})
+            continue
+        if rule:
+            left.append({"service": name, "reason": "placed by a constraint"})
+            continue
+        ok, out = run(["docker", "service", "update", "--force", "--detach=false", "--quiet", name],
+                      deadline=MOVE_DEADLINE_SECONDS)
+        if ok:
+            moved.append(name)
+        else:
+            failed.append(name)
+            left.append({"service": name, "reason": f"didn't settle: {out.strip()[-120:]}"})
+    detail = f"moved {len(moved)} of {len(services)}"
+    if failed:
+        detail += "; didn't settle: " + ", ".join(failed)
+    return {"ok": not failed, "moved": moved, "left": left, "detail": detail}
+
+
+# ── drive health (#5789) ─────────────────────────────────────────────────────
+# smartctl ≥ 7.0 answers in JSON, the same for SATA, SAS and NVMe drives on any
+# distribution; an older one is reported, never parsed as text. The helper
+# finds the drives itself: the request names none.
+
+SMARTCTL_MIN = (7, 0)
+SMARTCTL_DEADLINE_SECONDS = 60
+DRIVES_MAX = 32
+# smartctl's exit status is a bit mask: 1 the command line, 2 the device
+# couldn't be opened. Higher bits describe the disk, and its JSON still holds.
+_SMARTCTL_UNREAD = 0b11
+# ATA attributes counted as bad sectors.
+_ATA_BAD = {5: "reallocated", 197: "pending", 198: "offline_uncorrectable"}
+
+
+def smartctl_version():
+    """(major, minor) of the installed smartctl, or None when there is none."""
+    if not shutil.which("smartctl"):
+        return None
+    ok, out = run(["smartctl", "--version"], deadline=30)
+    m = re.search(r"smartctl (\d+)\.(\d+)", out)
+    return (int(m.group(1)), int(m.group(2))) if m else (0, 0)
+
+
+def smartctl_json(args: list):
+    """(exit status, parsed JSON or None) of one smartctl call."""
+    try:
+        done = subprocess.run(["smartctl", "--json", *args], capture_output=True, text=True,
+                              timeout=SMARTCTL_DEADLINE_SECONDS,
+                              env=dict(os.environ, LANG="C", LC_ALL="C"))
+    except (subprocess.TimeoutExpired, OSError):
+        return None, None
+    try:
+        return done.returncode, json.loads(done.stdout)
+    except ValueError:
+        return done.returncode, None
+
+
+def _int(v):
+    return v if isinstance(v, int) and not isinstance(v, bool) else None
+
+
+def drive_name(name: str, kind) -> str:
+    """A drive's name, unique on its host. Disks behind a RAID controller
+    share one path (/dev/bus/0) and differ only by the slot in their type
+    (megaraid,3; cciss,0), so the slot is part of the name (#5806)."""
+    base = os.path.basename(name)
+    if name.startswith("/dev/bus/"):
+        base = "bus" + base
+    if isinstance(kind, str) and "," in kind:
+        base += "-" + re.sub(r"[^A-Za-z0-9]", "", kind.split("+")[-1])
+    return base
+
+
+def drive_summary(name: str, status, data, kind=None) -> dict:
+    """The few figures Steward reads from one drive's `smartctl -a` JSON."""
+    out = {"device": drive_name(name, kind)}
+    if not isinstance(data, dict) or status is None or status & _SMARTCTL_UNREAD:
+        messages = ((data.get("smartctl") or {}).get("messages") or []) if isinstance(data, dict) else []
+        text = "; ".join(str(m.get("string", "")) for m in messages if isinstance(m, dict))
+        out["error"] = (text or "smartctl couldn't read it")[:200]
+        return out
+    dev = data.get("device") or {}
+    out["protocol"] = str(dev.get("protocol") or "")[:16]
+    out["model"] = str(data.get("model_name") or data.get("scsi_model_name") or "")[:80]
+    cap = (data.get("user_capacity") or {}).get("bytes")
+    if _int(cap):
+        out["capacity_bytes"] = cap
+    support = data.get("smart_support") or {}
+    if support.get("available") is False or support.get("enabled") is False:
+        out["available"] = False             # a virtual disk, or SMART switched off
+        return out
+    passed = (data.get("smart_status") or {}).get("passed")
+    if isinstance(passed, bool):
+        out["passed"] = passed
+    temp = _int((data.get("temperature") or {}).get("current"))
+    if temp is not None:
+        out["temp_c"] = temp
+    hours = _int((data.get("power_on_time") or {}).get("hours"))
+    if hours is not None:
+        out["power_on_hours"] = hours
+    nvme = data.get("nvme_smart_health_information_log") or {}
+    for src, dst in (("critical_warning", "critical_warning"), ("percentage_used", "life_used_pct"),
+                     ("media_errors", "media_errors"), ("available_spare", "spare_pct"),
+                     ("available_spare_threshold", "spare_threshold_pct")):
+        if _int(nvme.get(src)) is not None:
+            out[dst] = nvme[src]
+    for attr in (data.get("ata_smart_attributes") or {}).get("table") or []:
+        key = _ATA_BAD.get(attr.get("id")) if isinstance(attr, dict) else None
+        raw = _int((attr.get("raw") or {}).get("value")) if key else None
+        if raw is not None:
+            out[key] = raw
+    used = _int((data.get("endurance_used") or {}).get("current_percent"))          # ATA SSDs
+    used = used if used is not None else _int(data.get("scsi_percentage_used_endurance_indicator"))
+    if used is not None and "life_used_pct" not in out:
+        out["life_used_pct"] = used
+    if _int(data.get("scsi_grown_defect_list")) is not None:
+        out["grown_defects"] = data["scsi_grown_defect_list"]
+    return out
+
+
+def do_drives() -> dict:
+    """Each drive's health, as {"checked_at", "smartctl", "drives"} or
+    {"checked_at", "error"} when smartctl is missing or too old."""
+    base = {"checked_at": now_iso()}
+    version = smartctl_version()
+    if version is None:
+        return {**base, "error": "smartctl not installed"}
+    base["smartctl"] = "%d.%d" % version
+    if version < SMARTCTL_MIN:
+        return {**base, "error": "smartctl %d.%d is too old: drive health needs 7.0 or newer" % version}
+    _, scan = smartctl_json(["--scan-open"])
+    devices = (scan or {}).get("devices") if isinstance(scan, dict) else None
+    if not isinstance(devices, list):
+        return {**base, "error": "smartctl couldn't list the drives"}
+    drives = []
+    for d in devices[:DRIVES_MAX]:
+        name, kind = (d.get("name"), d.get("type")) if isinstance(d, dict) else (None, None)
+        if not isinstance(name, str) or not name.startswith("/dev/"):
+            continue
+        args = ["-a", name] if not isinstance(kind, str) else ["-a", "-d", kind, name]
+        status, data = smartctl_json(args)
+        drives.append(drive_summary(name, status, data, kind))
+    return {**base, "drives": drives}
+
+
+def write_drives(report: dict, path: str | None = None) -> None:
+    """DRIVES, written whole and readable by the agent."""
+    path = path or DRIVES
+    tmp = path + ".tmp"
+    with open(tmp, "w") as fh:
+        json.dump(report, fh)
+    os.chmod(tmp, 0o644)
+    os.replace(tmp, path)
+
+
+def write_result(result: dict) -> None:
+    """RESULTS/<id>.json, written whole (a temp file renamed into place)."""
+    os.makedirs(RESULTS, mode=0o755, exist_ok=True)
+    final = os.path.join(RESULTS, result["id"] + ".json")
+    tmp = final + ".tmp"
+    with open(tmp, "w") as fh:
+        json.dump(result, fh)
+    os.chmod(tmp, 0o644)
+    os.replace(tmp, final)
+
+
+def already_handled(job_id: str) -> bool:
+    """A request the agent sent again, for a job already run, is not run twice."""
+    return os.path.exists(os.path.join(RESULTS, job_id + ".json"))
+
+
+def handle(req: dict) -> dict:
+    base = {"id": req["id"], "action": req["action"], "started_at": now_iso()}
+    if req["action"] == "drives":               # a read, not a job: no result
+        write_drives(do_drives())
+        return {**base, "state": "done"}
+    write_result({**base, "state": "running"})
+    action = req["action"]
+    if action == "check":
+        outcome = do_check()
+    elif action == "update":
+        outcome = do_update()
+    elif action in ("drain", "undrain"):
+        outcome = do_node("drain" if action == "drain" else "active", req.get("node"))
+    elif action == "rebalance":
+        outcome = do_rebalance(req["services"])
+    else:                                            # reboot
+        result = {**base, "state": "rebooting", "ok": True, "finished_at": now_iso()}
+        write_result(result)
+        os.sync()
+        run(["systemctl", "reboot"], deadline=60)
+        return result
+    result = {**base, **outcome, "state": "done" if outcome.get("ok") else "failed",
+              "finished_at": now_iso()}
+    write_result(result)
+    return result
+
+
+def prune_results(now: float | None = None) -> None:
+    now = time.time() if now is None else now
+    try:
+        names = os.listdir(RESULTS)
+    except OSError:
+        return
+    for name in names:
+        path = os.path.join(RESULTS, name)
+        try:
+            if now - os.lstat(path).st_mtime > RESULTS_KEPT_SECONDS:
+                os.unlink(path)
+        except OSError:
+            pass
+
+
+def pending_requests() -> list:
+    """Request files, oldest first."""
+    try:
+        names = [n for n in os.listdir(REQUESTS) if n.endswith(".json")]
+    except OSError:
+        return []
+    paths = [os.path.join(REQUESTS, n) for n in names]
+
+    def age(p):
+        try:
+            return os.lstat(p).st_mtime
+        except OSError:
+            return 0.0
+    return sorted(paths, key=age)
+
+
+def main() -> int:
+    prune_results()
+    for path in pending_requests():
+        req = read_request(path)
+        try:
+            os.unlink(path)                 # whatever it held: never re-read in a loop
+        except OSError:
+            pass
+        if req is None:
+            sys.stderr.write(f"[steward-maint] ignored {os.path.basename(path)}: not a valid request\n")
+            continue
+        if already_handled(req["id"]):
+            continue
+        result = handle(req)
+        sys.stderr.write(f"[steward-maint] {req['action']} {req['id']}: {result['state']}\n")
+        if result["state"] == "rebooting":
+            break                           # the rest waits for the next boot
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

plugins/host_agent/files/steward-agent.service

@@ -0,0 +1,26 @@
+# Served by Steward at /plugins/host_agent/files/steward-agent.service and
+# installed by install.sh and the host_agent playbooks: one copy of the unit.
+[Unit]
+Description=Steward host agent
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=steward-agent
+Environment=STEWARD_AGENT_CONFIG=/etc/steward-agent.conf
+ExecStart=/usr/bin/env python3 /usr/local/lib/steward-agent/agent.py
+Restart=on-failure
+RestartSec=5
+# Stopping waits for the agent's last post, bounded at 10 s.
+TimeoutStopSec=30
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+ReadOnlyPaths=/proc /sys
+# The one place the agent writes: requests to the maintenance helper (#5766).
+ReadWritePaths=-/var/lib/steward-agent/requests
+
+[Install]
+WantedBy=multi-user.target

plugins/host_agent/files/steward-maint.path

@@ -0,0 +1,10 @@
+# Starts the maintenance helper when the host agent leaves a request (#5766).
+[Unit]
+Description=Steward maintenance requests from the host agent
+
+[Path]
+PathExistsGlob=/var/lib/steward-agent/requests/*.json
+Unit=steward-maint.service
+
+[Install]
+WantedBy=paths.target

plugins/host_agent/files/steward-maint.service

@@ -0,0 +1,11 @@
+# Runs as root, and does only what maint.py's fixed list allows: update the
+# host's packages, reboot it, drain / reactivate its swarm node (#5766), or
+# read its drives' health with smartctl (#5789).
+[Unit]
+Description=Steward maintenance helper
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/env python3 /usr/local/lib/steward-agent/maint.py
+# A full upgrade on a slow mirror; past this systemd stops it.
+TimeoutStartSec=3600

plugins/host_agent/maintenance.py

@@ -0,0 +1,660 @@
+# plugins/host_agent/maintenance.py
+"""A host's maintenance job, followed to its end through any restart (#5767).
+
+Steward plans and the host's agent acts: a job is one host's update and, only
+when the update says one is needed, its reboot, with the host's swarm node
+drained first and made active again after. Each action is one step the agent
+hands to its root helper (maint.py, #5766).
+
+A job first asks the helper what the update would change (#6098). Docker's
+own packages restart the daemon from their install scripts, stopping every
+container on the host, so a swarm node is drained before an update that
+upgrades Docker; otherwise only before a reboot. After a drain the node's
+tasks get DRAIN_SETTLE to move before the update or reboot goes ahead. The
+containers running outside the swarm when the job began must be running again
+when it ends, within RECOVER_LIMIT, or the job fails naming them.
+
+Everything lives on the job's row, so Steward restarting, or the host Steward
+runs on rebooting, loses nothing:
+the next post from the agent carries the step's result, and the `started`
+event a host sends after booting is what completes a reboot.
+
+The transitions here are model-free (a job is anything with the attributes),
+so they're unit-tested without a database; routes.ingest and the scheduler's
+tick apply them.
+
+States: queued → [checking →] [draining → settling →] updating →
+[draining → settling →] [rebooting →] [undraining →] [recovering →] done, or
+failed / timed_out. A node is drained at most once a job: before the update
+when Docker is upgraded, else before the reboot. settling and recovering are
+Steward's own waits, moved on by the minute tick. An agent older than
+CHECK_AGENT gets no check: its job starts at updating.
+
+A job that fails or is given up while its node may be drained makes the node
+active again first: it goes to undraining with the outcome kept in
+`ends_as`, and finishes as that once the undrain is through. A job never ends
+leaving behind a drain it made.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+ACTIVE = ("queued", "checking", "draining", "settling", "updating", "rebooting",
+          "undraining", "recovering")
+FINISHED = ("done", "failed", "timed_out")
+# How a job failed for not coming back into service begins (cadence.py): its
+# update (and reboot) went through, so once the host is back nothing is left
+# failed.
+NOT_BACK = "not back in service within"
+# The helper action each handed state asks for.
+STEP_ACTION = {"checking": "check", "updating": "update", "draining": "drain",
+               "rebooting": "reboot", "undraining": "undrain"}
+# How long a state may last before the job is given up. Queued counts from the
+# job's creation (the host may be silent); the rest from their hand-off.
+DEADLINES = {
+    "queued": timedelta(hours=1),
+    "checking": timedelta(minutes=30),     # two listings, the helper stops each at 10
+    # The drain's result comes back the way the drain can take down: draining
+    # the node that runs the proxy (or Steward) moves them, and the result
+    # waits until they run elsewhere, image pull included.
+    "draining": timedelta(minutes=30),
+    "settling": timedelta(minutes=15),     # the tick moves it on at DRAIN_SETTLE
+    "updating": timedelta(minutes=90),     # the helper itself stops at 60
+    "rebooting": timedelta(minutes=30),
+    "undraining": timedelta(minutes=10),
+    "recovering": timedelta(minutes=15),   # the tick fails it at RECOVER_LIMIT
+}
+# After a drain, how long the node's swarm tasks get to move before the
+# update or the reboot goes ahead (the operator, #6098: "drain wait 5 minutes
+# and then reboot").
+DRAIN_SETTLE = timedelta(minutes=5)
+# How long the containers that ran outside the swarm before the job get to
+# run again after it.
+RECOVER_LIMIT = timedelta(minutes=10)
+# The first agent whose helper answers a check (#6098).
+CHECK_AGENT = (1, 24, 0)
+# A restart policy under which Docker starts a container again after a reboot.
+RESTARTS = ("always", "unless-stopped")
+CONTAINERS_KEPT = 40
+# A host's clock and Steward's differ by seconds; a boot this far before the
+# reboot was handed out is still taken as that reboot.
+BOOT_SLACK = timedelta(seconds=60)
+DETAIL_MAX = 500
+
+
+def step_id(job, state: str) -> str:
+    """The helper request id for a job's step: unique per job and step, and in
+    the helper's id alphabet (letters, digits, dashes)."""
+    return f"{job.id}-{STEP_ACTION[state]}"
+
+
+def _enter(job, state: str, now: datetime) -> None:
+    job.state, job.state_at = state, now
+    job.step_id = step_id(job, state) if state in STEP_ACTION else None
+    job.step_seen = False               # a new step is handed out until its result arrives
+    if state in FINISHED:
+        job.finished_at = now
+
+
+def _finish(job, state: str, now: datetime, detail: str | None = None) -> None:
+    if detail:
+        job.detail = detail[:DETAIL_MAX]
+    _enter(job, state, now)
+
+
+def may_be_drained(job) -> bool:
+    """Whether the job may have left its node drained: its drain was handed
+    out (a result may just not have come back), or went through and the node
+    hasn't been made active again."""
+    return drains(job) and (job.state == "draining" or
+                            (job.drained_at is not None and job.state in ("settling", "updating", "rebooting")))
+
+
+def _give_up(job, state: str, now: datetime, detail: str) -> None:
+    """End the job as `state` (failed or timed_out), making its node active
+    again first when the job may have drained it."""
+    if may_be_drained(job):
+        job.detail = detail[:DETAIL_MAX]
+        job.ends_as = state
+        _enter(job, "undraining", now)
+    else:
+        _finish(job, state, now, detail)
+
+
+def can_check(agent_version) -> bool:
+    """Whether an agent this version hands the helper a check."""
+    try:
+        return tuple(int(x) for x in str(agent_version).split(".")[:3]) >= CHECK_AGENT
+    except ValueError:
+        return False
+
+
+def start(job, now: datetime, check: bool = False) -> None:
+    """The host's agent is about to be handed the job: what the update would
+    change first, when its helper can say, else the update."""
+    if job.state == "queued":
+        _enter(job, "checking" if check else "updating", now)
+
+
+def drains(job) -> bool:
+    """Whether the job drains the host's swarm node: not when the host isn't
+    in a swarm, nor when the operator keeps its node drained."""
+    return bool(job.node_id and job.drain_host_id)
+
+
+def outside_swarm(job) -> list:
+    """The containers the check found running outside the swarm."""
+    return (job.checked or {}).get("containers") or []
+
+
+def executor(job) -> str:
+    """The host whose agent carries out the current step: the job's own host,
+    except a drain or undrain, which a swarm manager does."""
+    if job.state in ("draining", "undraining") and job.drain_host_id:
+        return job.drain_host_id
+    return job.host_id
+
+
+def handoff(job) -> dict | None:
+    """What to hand the executing agent now, or None. Handed on every post
+    until a result for the step arrives; the agent and the helper both ignore
+    an id they've already taken, so a repeat is harmless."""
+    if job.state not in STEP_ACTION or job.step_seen:
+        return None
+    out = {"id": job.step_id, "action": STEP_ACTION[job.state]}
+    if job.state in ("draining", "undraining") and job.node_id:
+        out["node"] = job.node_id
+    return out
+
+
+REBOOT_POLICIES = ("if_required", "always", "never")
+
+
+def wants_reboot(job) -> bool:
+    """Reboot only when the update says one is needed (the operator's
+    default), always (a plan may ask), or never."""
+    if job.reboot_policy == "always":
+        return True
+    return bool(job.reboot_needed) and job.reboot_policy == "if_required"
+
+
+def _checked(event: dict) -> dict:
+    """What a check said, kept on the job: bounded, and only the fields used."""
+    containers = []
+    for c in event.get("containers") or []:
+        if isinstance(c, dict) and isinstance(c.get("name"), str) and c["name"]:
+            containers.append({"name": c["name"][:64], "restart": str(c.get("restart") or "no")[:16]})
+    packages = [str(p)[:64] for p in event.get("docker_packages") or [] if isinstance(p, str)]
+    pending = event.get("pending")
+    return {"pending": pending if isinstance(pending, int) else None,
+            "docker_packages": packages[:10], "containers": containers[:CONTAINERS_KEPT]}
+
+
+def _after_check(job, event: dict, now: datetime) -> None:
+    job.docker_update = event.get("docker_update") is True
+    job.checked = _checked(event)
+    _enter(job, "draining" if job.docker_update and drains(job) else "updating", now)
+
+
+def _after_update(job, now: datetime) -> None:
+    if not wants_reboot(job):
+        _after_work(job, now)
+    elif drains(job) and job.drained_at is None:
+        _enter(job, "draining", now)
+    else:
+        _enter(job, "rebooting", now)
+
+
+def _after_work(job, now: datetime) -> None:
+    """The update (and any reboot) is over: make the node active again if the
+    job drained it, then see the containers back."""
+    if job.drained_at is not None and drains(job):
+        _enter(job, "undraining", now)
+    else:
+        _recover(job, now)
+
+
+def _recover(job, now: datetime) -> None:
+    _enter(job, "recovering" if outside_swarm(job) else "done", now)
+
+
+def apply_result(job, event: dict, now: datetime) -> bool:
+    """Apply a `maintenance` event for this job's current step; True when the
+    job changed. A result for a step the job has moved past is ignored, so a
+    repeated or late report does nothing."""
+    if job.state not in STEP_ACTION or event.get("id") != job.step_id:
+        return False
+    state = event.get("state")
+    if state in ("running", "rebooting"):
+        if job.step_seen:
+            return False
+        job.step_seen = True
+        return True
+    if state not in ("done", "failed"):
+        return False
+    if state == "failed" or event.get("ok") is False:
+        what = STEP_ACTION[job.state]
+        detail = f"{what} failed: {event.get('detail') or 'no detail'}"
+        if job.state == "undraining":
+            _finish(job, "failed", now, _after_given_up(job, detail + " (the node is still drained)"))
+        else:
+            _give_up(job, "failed", now, detail)
+        return True
+    if job.state == "checking":
+        _after_check(job, event, now)
+    elif job.state == "updating":
+        if isinstance(event.get("packages"), int):
+            job.packages = event["packages"]
+        job.reboot_needed = event.get("reboot_required") is True
+        _after_update(job, now)
+    elif job.state == "draining":
+        job.drained_at = now
+        _enter(job, "settling", now)
+    elif job.state == "undraining":
+        if job.ends_as:                 # given up: the node is active again, the job ends as it was
+            _enter(job, job.ends_as, now)
+        else:
+            _recover(job, now)
+    else:                               # rebooting: the boot completes it, not this
+        return False
+    return True
+
+
+def apply_boot(job, boot_at: datetime | None, now: datetime) -> bool:
+    """A `started` event from the job's host. A boot after the reboot was
+    handed out is that reboot: the host is back."""
+    if job.state != "rebooting" or boot_at is None or job.state_at is None:
+        return False
+    if boot_at < job.state_at - BOOT_SLACK:
+        return False                    # the agent restarted; the host didn't
+    job.back_at = now
+    _after_work(job, now)
+    return True
+
+
+def settle(job, now: datetime) -> bool:
+    """A drained node has had DRAIN_SETTLE for its tasks to move: on to the
+    update, or to the reboot when the update is done (reboot_needed is set)."""
+    if job.state != "settling" or job.state_at is None or now - job.state_at < DRAIN_SETTLE:
+        return False
+    _enter(job, "updating" if job.reboot_needed is None else "rebooting", now)
+    return True
+
+
+def recovered(job, running: set, now: datetime) -> bool:
+    """Done once every container that ran outside the swarm before the job
+    runs again (`running`: the host's containers seen running since the job
+    came to this state); failed, naming the rest, past RECOVER_LIMIT."""
+    if job.state != "recovering" or job.state_at is None:
+        return False
+    missing = [c for c in outside_swarm(job) if c["name"] not in running]
+    if not missing:
+        _enter(job, "done", now)
+        return True
+    if now - job.state_at < RECOVER_LIMIT:
+        return False
+    names = [c["name"] + ("" if c["restart"] in RESTARTS else " (no restart policy)")
+             for c in missing[:6]]
+    more = f" and {len(missing) - 6} more" if len(missing) > 6 else ""
+    _finish(job, "failed", now, f"not running again {int(RECOVER_LIMIT.total_seconds() // 60)} min "
+                                f"after the job: {', '.join(names)}{more}")
+    return True
+
+
+def check_deadline(job, now: datetime) -> bool:
+    """Give up on a job whose state has lasted past its deadline."""
+    limit = DEADLINES.get(job.state)
+    since = job.state_at or job.created_at
+    if limit is None or since is None or now - since <= limit:
+        return False
+    waiting = {"queued": "the host's agent didn't check in",
+               "checking": "the check of what the update would change didn't finish",
+               "draining": "the node wasn't drained",
+               "settling": "the wait after the drain didn't end",
+               "updating": "the update didn't finish",
+               "rebooting": "the host didn't come back",
+               "undraining": "the node wasn't made active again",
+               "recovering": "its containers weren't seen again"}[job.state]
+    detail = f"{waiting} within {int(limit.total_seconds() // 60)} min"
+    if job.state == "undraining":
+        _finish(job, job.ends_as or "timed_out", now, _after_given_up(job, detail))
+    else:
+        _give_up(job, "timed_out", now, detail)
+    return True
+
+
+def _after_given_up(job, detail: str) -> str:
+    """What an undrain going wrong adds to why a given-up job ended."""
+    return f"{job.detail}; then {detail}" if job.ends_as and job.detail else detail
+
+
+def window_end(job, now: datetime) -> datetime:
+    """When the job's maintenance window should end: the longest the rest of
+    the job could take, or now once it has finished."""
+    if job.state in FINISHED:
+        return now
+    left = sum((DEADLINES[s] for s in ACTIVE[ACTIVE.index(job.state):]), timedelta())
+    return now + left
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+# ── applied to the database: ingest, the tick, and starting a job ────────────
+
+async def _window(session, job, host_name: str, now: datetime) -> None:
+    """Keep the job's maintenance window over the host until the job ends: one
+    scoped to this host's agent metrics, so an alert the job causes waits.
+    A swarm node's job keeps a second over the swarm-wide figures (#5788):
+    draining and rebooting a node takes it down and can leave a service
+    pinned to it short, which is the job, not news."""
+    from steward.models.alerts import MaintenanceWindow
+    if job.window_id:
+        w = await session.get(MaintenanceWindow, job.window_id)
+        if w is not None:
+            w.end_at = max(w.start_at, window_end(job, now))
+    elif job.created_by is None or job.state in FINISHED:
+        return                          # a window needs an owner; none for a finished job
+    else:
+        w = MaintenanceWindow(name=f"Maintenance: {host_name}"[:128], start_at=now,
+                              end_at=window_end(job, now), scope_source="host_agent",
+                              scope_resource=host_name, created_by=job.created_by)
+        session.add(w)
+        await session.flush()
+        job.window_id = w.id
+    if job.node_id:
+        await _swarm_window(session, job, host_name, now)
+
+
+SWARM_SCOPE = ("docker", "swarm")       # the swarm-wide figures' source and resource
+
+
+async def _swarm_window(session, job, host_name: str, now: datetime) -> None:
+    """The swarm half of a swarm node's job window, found again by its name
+    and the job's start, so it needs no column of its own."""
+    from sqlalchemy import select
+    from steward.models.alerts import MaintenanceWindow
+    name = f"Maintenance: {host_name} (swarm)"[:128]
+    w = (await session.execute(select(MaintenanceWindow).where(
+        MaintenanceWindow.name == name, MaintenanceWindow.scope_source == SWARM_SCOPE[0],
+        MaintenanceWindow.scope_resource == SWARM_SCOPE[1],
+        MaintenanceWindow.start_at >= job.created_at))).scalars().first()
+    if w is not None:
+        w.end_at = max(w.start_at, window_end(job, now))
+    elif job.created_by is not None and job.state not in FINISHED:
+        session.add(MaintenanceWindow(name=name, start_at=now, end_at=window_end(job, now),
+                                      scope_source=SWARM_SCOPE[0], scope_resource=SWARM_SCOPE[1],
+                                      created_by=job.created_by))
+
+
+async def _host_name(session, host_id: str) -> str:
+    from steward.models.hosts import Host
+    host = await session.get(Host, host_id)
+    return host.name if host is not None else host_id
+
+
+async def _moved(session, job, now: datetime) -> None:
+    """After a job moved: its window follows it, and once it ends it goes on
+    the timeline."""
+    host_name = await _host_name(session, job.host_id)
+    await _window(session, job, host_name, now)
+    if job.state in FINISHED:
+        from . import timeline
+        await timeline.write(session, f"maintenance job {job.id}",
+                             timeline.job_entry(session, job, host_name))
+
+
+async def apply_events(session, host_id: str, events, now: datetime) -> int:
+    """Apply one post's `maintenance` results and `started` boots to the jobs
+    they belong to; returns how many jobs changed. A result is matched by its
+    step id, so a drain reported by a manager reaches the job of the node it
+    drained. A result no job owns may be a run's rebalance (rebalance.py)."""
+    from sqlalchemy import select
+    from . import rebalance
+    from .downtime import _at
+    from .models import HostMaintenanceJob
+    if not isinstance(events, list):
+        return 0
+    changed = set()
+    for event in events:
+        if not isinstance(event, dict):
+            continue
+        if event.get("kind") == "maintenance" and isinstance(event.get("id"), str):
+            job = (await session.execute(select(HostMaintenanceJob).where(
+                HostMaintenanceJob.step_id == event["id"],
+                HostMaintenanceJob.state.in_(ACTIVE)))).scalar_one_or_none()
+            if job is not None and apply_result(job, event, now):
+                changed.add(job)
+            elif job is None:
+                await rebalance.apply_event(session, event, now)
+        elif event.get("kind") == "started":
+            boot_at = _at(event.get("boot_at"))
+            for job in (await session.execute(select(HostMaintenanceJob).where(
+                    HostMaintenanceJob.host_id == host_id,
+                    HostMaintenanceJob.state == "rebooting"))).scalars():
+                if apply_boot(job, boot_at, now):
+                    changed.add(job)
+    for job in changed:
+        await _moved(session, job, now)
+    return len(changed)
+
+
+async def handoffs(session, host_id: str, host_name: str, now: datetime,
+                   agent_version: str | None = None) -> list[dict]:
+    """The steps this host's agent should carry out now. A queued job for the
+    host starts here, when its agent is known to be listening, with a check
+    when the agent is new enough to hand one over. A manager also carries out
+    a run's rebalance."""
+    from sqlalchemy import or_, select
+    from . import rebalance
+    from .models import HostMaintenanceJob
+    jobs = (await session.execute(select(HostMaintenanceJob).where(
+        HostMaintenanceJob.state.in_(ACTIVE),
+        or_(HostMaintenanceJob.host_id == host_id,
+            HostMaintenanceJob.drain_host_id == host_id)
+    ).order_by(HostMaintenanceJob.created_at))).scalars().all()
+    out = []
+    for job in jobs:
+        if job.state == "queued" and job.host_id == host_id:
+            start(job, now, check=can_check(agent_version))
+            await _window(session, job, host_name, now)
+        if executor(job) == host_id:
+            step = handoff(job)
+            if step is not None:
+                out.append(step)
+    return out + await rebalance.handoffs(session, host_id)
+
+
+async def _running(session, job) -> set:
+    """The job's host's containers seen running since the job came to its
+    state; without the Docker plugin, the ones it waits for, as nothing can
+    say otherwise."""
+    from sqlalchemy import select
+    models = docker_models()
+    if models is None:
+        return {c["name"] for c in outside_swarm(job)}
+    DockerContainer = models.DockerContainer
+    return set((await session.execute(select(DockerContainer.name).where(
+        DockerContainer.host_id == job.host_id, DockerContainer.status == "running",
+        DockerContainer.scraped_at >= job.state_at))).scalars())
+
+
+async def tick(session, now: datetime) -> int:
+    """Move on the jobs waiting on Steward (the wait after a drain, the
+    containers seen again) and give up on jobs past their deadline; returns
+    how many were given up."""
+    from sqlalchemy import select
+    from .models import HostMaintenanceJob
+    gave_up = 0
+    for job in (await session.execute(select(HostMaintenanceJob).where(
+            HostMaintenanceJob.state.in_(ACTIVE)))).scalars():
+        moved = settle(job, now)
+        if job.state == "recovering":
+            moved = recovered(job, await _running(session, job), now) or moved
+        if check_deadline(job, now):
+            gave_up += 1
+            moved = True
+        if moved:
+            await _moved(session, job, now)
+    return gave_up
+
+
+class CannotStart(Exception):
+    """Why a host can't take a maintenance job now, in words for the page."""
+
+
+def pick_drainer(node_hostname_to_id: dict, managers: dict, host_name: str,
+                 host_id: str, helpers: set, availability: str = "active") -> tuple:
+    """(node_id, drain_host_id) for a host in a swarm, (None, None) for one
+    that isn't. `managers` maps a manager's host id to its swarm node id;
+    `helpers` is the hosts whose agent has the maintenance helper. The host
+    drains itself when it is a manager, else another manager does. A node the
+    operator already set to drain or pause (a manager kept free of work) is
+    left as it is: no drain before, and no making it active after."""
+    node_id = node_hostname_to_id.get(host_name.lower())
+    if node_id is None:
+        return None, None
+    if availability != "active":
+        return node_id, None
+    if host_id in managers and host_id in helpers:
+        return node_id, host_id
+    for mgr in sorted(managers):
+        if mgr != host_id and mgr in helpers:
+            return node_id, mgr
+    raise CannotStart("it is a swarm node and no swarm manager has the maintenance "
+                      "helper to drain it: run Update agent on a manager")
+
+
+async def plan_job(session, host, reg, reboot_policy: str = "if_required") -> tuple:
+    """(node_id, drain_host_id) for a job on `host`, or CannotStart saying why
+    it can't have one now. The panel asks this before offering the button."""
+    from sqlalchemy import select
+    from .models import HostAgentRegistration, HostMaintenanceJob
+    if reg is None or not reg.maint_helper:
+        raise CannotStart("its agent has no maintenance helper: run Update agent first")
+    if reboot_policy not in REBOOT_POLICIES:
+        raise CannotStart(f"unknown reboot policy {reboot_policy!r}")
+    busy = (await session.execute(select(HostMaintenanceJob.id).where(
+        HostMaintenanceJob.host_id == host.id,
+        HostMaintenanceJob.state.in_(ACTIVE)))).first()
+    if busy is not None:
+        raise CannotStart("a maintenance job is already running on it")
+    nodes, managers, availability = await _swarm(session)
+    helpers = set((await session.execute(select(HostAgentRegistration.host_id).where(
+        HostAgentRegistration.maint_helper.is_(True)))).scalars())
+    node_id = nodes.get(host.name.lower())
+    try:
+        return pick_drainer(nodes, managers, host.name, host.id, helpers,
+                            availability.get(node_id, "active"))
+    except CannotStart:
+        # A job that never reboots needs a drain only if Docker is upgraded;
+        # with no manager to drain, it runs undrained rather than not at all.
+        if reboot_policy == "never":
+            return None, None
+        raise
+
+
+async def create_job(session, host, reg, user_id: str | None,
+                     reboot_policy: str = "if_required", now: datetime | None = None,
+                     run_id: str | None = None):
+    """Queue a maintenance job for `host`; it starts with the agent's next post."""
+    from .models import HostMaintenanceJob
+    now = now or utcnow()
+    node_id, drain_host_id = await plan_job(session, host, reg, reboot_policy)
+    job = HostMaintenanceJob(host_id=host.id, state="queued", state_at=now,
+                             reboot_policy=reboot_policy, node_id=node_id,
+                             drain_host_id=drain_host_id, created_by=user_id, created_at=now,
+                             run_id=run_id)
+    session.add(job)
+    await session.flush()
+    return job
+
+
+async def latest_job(session, host_id: str, now: datetime, days: int = 366):
+    """The host's running job, else its last one (within a year), for the panel."""
+    from sqlalchemy import select
+    from .models import HostMaintenanceJob
+    return (await session.execute(select(HostMaintenanceJob).where(
+        HostMaintenanceJob.host_id == host_id,
+        HostMaintenanceJob.created_at >= now - timedelta(days=days),
+    ).order_by(HostMaintenanceJob.state.in_(ACTIVE).desc(),
+               HostMaintenanceJob.created_at.desc()).limit(1))).scalar_one_or_none()
+
+
+LABELS = {
+    "queued": "Waiting for the agent",
+    "checking": "Checking what the update will change",
+    "draining": "Draining its swarm node",
+    "settling": "Giving its swarm tasks 5 min to move",
+    "updating": "Updating",
+    "rebooting": "Rebooting",
+    "undraining": "Making its swarm node active again",
+    "recovering": "Waiting for its containers to run again",
+    "done": "Done",
+    "failed": "Failed",
+    "timed_out": "Timed out",
+}
+
+
+def came_back_late(job) -> bool:
+    """A job failed only for not coming back into service in time, whose
+    host has since been found back (#6175)."""
+    return (job.state == "failed" and job.came_back_at is not None
+            and (job.detail or "").startswith(NOT_BACK))
+
+
+def still_failed(job) -> bool:
+    """Whether a finished job still flags its host: it failed or timed out,
+    and isn't a late return whose host has since come back (#6175). The
+    maintenance_failed figure, the overview and the pages all ask this."""
+    return job.state in ("failed", "timed_out") and not came_back_late(job)
+
+
+def summary(job) -> str:
+    """One line on what the job did, for the panel."""
+    if job.state not in FINISHED:
+        return LABELS[job.state]
+    if came_back_late(job):
+        return f"Back in service late, {job.came_back_at:%Y-%m-%d %H:%M}Z"
+    parts = []
+    if job.docker_update:
+        parts.append("Docker upgraded")
+    if job.packages is not None:
+        parts.append(f"{job.packages} package{'s' if job.packages != 1 else ''} updated")
+    if job.back_at is not None:
+        parts.append("rebooted")
+    elif job.reboot_needed and job.reboot_policy == "never":
+        parts.append("reboot needed, not allowed by this job")
+    elif job.state == "done" and job.reboot_needed is False:
+        parts.append("no reboot needed")
+    head = LABELS[job.state]
+    return f"{head}: {', '.join(parts)}" if parts else head
+
+
+def docker_models():
+    """The loaded Docker plugin's models module, or None when it isn't loaded."""
+    from steward.core.plugin_manager import loaded_plugin_submodule
+    return loaded_plugin_submodule("docker", "models")
+
+
+async def _swarm(session) -> tuple:
+    """({node hostname: node id}, {manager host id: its node id},
+    {node id: availability}) from what the Docker plugin last stored; empty
+    when it isn't installed. A manager is a host whose agent reported the
+    swarm's nodes."""
+    models = docker_models()
+    if models is None:
+        return {}, {}, {}
+    DockerSwarmNode = models.DockerSwarmNode
+    from sqlalchemy import select
+    from steward.models.hosts import Host
+    rows = (await session.execute(select(DockerSwarmNode))).scalars().all()
+    nodes = {(n.hostname or "").lower(): n.node_id for n in rows if n.hostname}
+    reporters = {n.host_id for n in rows}
+    names = dict((await session.execute(select(Host.id, Host.name).where(
+        Host.id.in_(reporters)))).all()) if reporters else {}
+    managers = {hid: nodes.get(name.lower()) for hid, name in names.items()}
+    availability = {n.node_id: (n.availability or "active")
+                    for n in sorted(rows, key=lambda n: n.updated_at)}
+    return nodes, managers, availability

plugins/host_agent/mcp_tools.py

@@ -5,14 +5,18 @@ from datetime import timedelta
 
 from sqlalchemy import select
 
+import json
+
 from steward.hosts import service as host_svc
-from steward.mcp._context import mcp_app
+from steward.mcp._context import current_caller, mcp_app
-from steward.mcp.tools._shape import age_s, db_session, iso, now_utc
+from steward.mcp.tools._audit import audit_mcp
+from steward.mcp.tools._shape import age_s, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd
 from steward.mcp.tools.overview import finding
 from steward.models.hosts import Host
 from steward.models.users import UserRole
 
-from .models import HostAgentRegistration
+from . import cadence, maintenance, rebalance, senders
+from .models import HostAgentRegistration, HostMaintenanceJob, HostMaintenancePlan, HostMaintenanceRun
 
 # A registration this old that has never checked in is a failed install, not one in progress.
 PENDING_GRACE = timedelta(minutes=30)
@@ -33,6 +37,19 @@ async def _agents(db):
     )).all()
 
 
+def _updates(reg) -> dict | None:
+    """A registration's pending updates and reboot, as list_agents says them (#5765)."""
+    if reg.updates_checked_at is None and reg.reboot_required is None:
+        return None
+    return {
+        "manager": reg.updates_manager, "pending": reg.updates_pending,
+        "security": reg.updates_security, "checked_at": iso(reg.updates_checked_at),
+        "error": reg.updates_error, "reboot_required": reg.reboot_required,
+        "reboot_required_since": iso(reg.reboot_required_since),
+        "reboot_packages": reg.reboot_packages.split(", ") if reg.reboot_packages else [],
+    }
+
+
 async def list_agents() -> dict:
     """Every host agent Steward has registered, with its version and whether it is reporting.
 
@@ -41,6 +58,11 @@ async def list_agents() -> dict:
     (registered, never checked in, usually an install that is still running
     or that failed). Each row carries the agent version, kernel, distro, arch
     and the IP the agent reports. get_host shows an agent's latest metrics.
+
+    `updates` is what the host's package manager has waiting, from the
+    agent's hourly check (agent ≥ 1.16.0): manager, pending, security (null
+    when the manager can't say), checked_at, error, and reboot_required with
+    since when and the packages that asked for it. null before the first check.
     """
     now = now_utc()
     stale_after = host_svc.agent_stale_after_seconds(mcp_app().config)
@@ -54,6 +76,7 @@ async def list_agents() -> dict:
             "agent_version": reg.agent_version, "kernel": reg.kernel, "distro": reg.distro,
             "arch": reg.arch, "reported_ip": reg.host_ip,
             "registered_at": iso(reg.created_at),
+            "updates": _updates(reg),
         }
         for reg, h in rows
     ]
@@ -61,21 +84,300 @@ async def list_agents() -> dict:
     return {"stale_after_s": stale_after, "counts": counts, "agents": agents, "total": len(agents)}
 
 
+# ── maintenance (#5834): plans, runs, jobs, and starting one ─────────────────
+
+JOBS_MAX = 50
+# A failed job stays on the overview this long, unless a later one succeeded.
+FAILED_JOB_DAYS = 7
+
+
+async def _names(db, ids) -> dict[str, str]:
+    from steward.models.users import User
+    ids = {i for i in ids if i}
+    if not ids:
+        return {}
+    hosts = dict((await db.execute(select(Host.id, Host.name).where(Host.id.in_(ids)))).all())
+    users = dict((await db.execute(select(User.id, User.username).where(User.id.in_(ids)))).all())
+    return {**users, **hosts}
+
+
+def _job_row(job, names: dict, now) -> dict:
+    deadline = maintenance.DEADLINES.get(job.state)
+    started = job.created_at if job.state == "queued" else job.state_at
+    return {
+        "job_id": job.id,
+        "host": names.get(job.host_id, job.host_id),
+        "state": job.state,
+        "label": maintenance.LABELS.get(job.state, job.state),
+        "summary": maintenance.summary(job),
+        "active": job.state in maintenance.ACTIVE,
+        "reboot_policy": job.reboot_policy,
+        "packages_updated": job.packages,
+        "reboot_needed": job.reboot_needed,
+        "rebooted_at": iso(job.back_at),
+        "swarm_node": job.node_id,
+        "drained_by": names.get(job.drain_host_id, job.drain_host_id),
+        "drained_at": iso(job.drained_at),
+        "came_back_at": iso(job.came_back_at),
+        "ends_as": job.ends_as,
+        "docker_update": job.docker_update,
+        "checked": job.checked,
+        "detail": job.detail,
+        "state_since": iso(job.state_at), "state_age_s": age_s(job.state_at, now),
+        "gives_up_at": iso(started + deadline) if deadline and started else None,
+        "started_by": names.get(job.created_by) if job.created_by else
+                      ("maintenance plan" if job.run_id else None),
+        "run_id": job.run_id,
+        "created_at": iso(job.created_at),
+        "finished_at": iso(job.finished_at),
+    }
+
+
+async def list_maintenance_plans() -> dict:
+    """Each maintenance plan: which inventory group it updates, when (weekly
+    slot, time zone, how long a run may keep starting hosts), its reboot
+    policy, whether it is enabled, its next slot, and its latest run.
+
+    A run updates the group's hosts one at a time; `latest_run` says how far
+    it got (`hosts_done` of `hosts`), which hosts it passed over and why
+    (`skipped`), and how it ended. `last_slot_at` is the last scheduled slot
+    that fired, null until one has. get_maintenance_job shows one host's job.
+    Plans are created and edited on the Maintenance page.
+
+    With `spread_after_run`, a run that drained swarm nodes ends by spreading
+    the swarm's services back out, leaving `keep_in_place` where they are;
+    `latest_run.spread` says what it moved and what it left and why (state:
+    handed, done, failed, timed_out, or nothing to move), null when the run
+    had none.
+    """
+    from steward.models.ansible_inventory import AnsibleGroup
+    now = now_utc()
+    async with db_session() as db:
+        plans = list((await db.execute(select(HostMaintenancePlan)
+                                       .order_by(HostMaintenancePlan.name))).scalars())
+        groups = dict((await db.execute(select(AnsibleGroup.id, AnsibleGroup.name))).all())
+        latest = {}
+        for plan in plans:
+            latest[plan.id] = (await db.execute(
+                select(HostMaintenanceRun).where(HostMaintenanceRun.plan_id == plan.id)
+                .order_by(HostMaintenanceRun.started_at.desc()).limit(1))).scalar_one_or_none()
+        names = await _names(db, [h for r in latest.values() if r
+                                  for h in json.loads(r.host_ids or "[]")
+                                  + [x["host_id"] for x in json.loads(r.skipped or "[]")]])
+    out = []
+    for plan in plans:
+        run = latest[plan.id]
+        nxt = cadence.next_slot(plan, now) if plan.enabled else None
+        row = {
+            "plan_id": plan.id, "name": plan.name,
+            "group": groups.get(plan.group_id, plan.group_id),
+            "schedule": cadence.describe(plan),
+            "weekday": cadence.WEEKDAYS[plan.weekday], "at_time": plan.at_time,
+            "time_zone": plan.time_zone, "window_minutes": plan.window_minutes,
+            "reboot_policy": plan.reboot_policy, "enabled": plan.enabled,
+            "spread_after_run": plan.spread,
+            "keep_in_place": rebalance.keep_list(plan.keep_in_place),
+            "next_slot_at": iso(nxt), "last_slot_at": iso(cadence.fired_slot(plan)),
+            "latest_run": None,
+        }
+        if run:
+            hosts = json.loads(run.host_ids or "[]")
+            row["latest_run"] = {
+                "run_id": run.id, "state": run.state, "manual": run.manual,
+                "hosts": [names.get(h, h) for h in hosts],
+                "hosts_done": min(run.position, len(hosts)),
+                "skipped": [{"host": names.get(x["host_id"], x["host_id"]), "reason": x["reason"]}
+                            for x in json.loads(run.skipped or "[]")],
+                "detail": run.detail,
+                "spread": _spread_row(run.rebalance),
+                "started_at": iso(run.started_at), "finished_at": iso(run.finished_at),
+            }
+        out.append(row)
+    return {"plans": out, "total": len(out)}
+
+
+def _spread_row(rb: dict | None) -> dict | None:
+    """A run's rebalance as the tool shows it: no step ids or window ids."""
+    if not rb:
+        return None
+    return {"state": rb.get("state"), "planned": rb.get("planned") or [],
+            "moved": rb.get("moved") or [], "left": rb.get("left") or [],
+            "detail": rb.get("detail"), "started_at": rb.get("at"),
+            "finished_at": rb.get("finished_at")}
+
+
+async def get_maintenance_job(host: str = "", job: str = "", limit: int = 10) -> dict:
+    """A host's maintenance jobs, newest first (its running one first), or one
+    job by id.
+
+    host: a host's name or id (list_hosts); job: a job id (from here, or a
+    plan's run). One of the two.
+    limit: how many of the host's jobs (default 10, max 50).
+
+    A job is one host's update and, only when the update needs it, a reboot,
+    with its swarm node drained first and made active again after. It first
+    checks what the update would change: `docker_update` is whether it
+    upgrades Docker (whose packages restart the daemon), and `checked` holds
+    {pending, docker_packages, containers}, the containers then running
+    outside the swarm. A swarm node is drained before an update that upgrades
+    Docker, else before a reboot; `drained_at` is when, and its tasks get 5
+    min to move (settling). `state` moves queued → [checking →] [draining →
+    settling →] updating → [draining → settling →] [rebooting →]
+    [undraining →] [recovering →] done, or failed / timed_out; a job given
+    up while its node may be drained goes through undraining first, and
+    `ends_as` is what it then finishes as. recovering
+    waits up to 10 min for those containers to run again; `detail` says why
+    a job failed, naming any that didn't. A job failed only for not coming
+    back into service in time has `came_back_at`, when its host was first
+    found back; from then it no longer flags the host. `summary` is the host page's line
+    ("Done: 12 packages updated, rebooted"). `gives_up_at` is when an active
+    job's current state times out.
+    """
+    if bool(host.strip()) == bool(job.strip()):
+        raise ValueError("pass either `host` (a name or id) or `job` (a job id).")
+    now = now_utc()
+    limit = clamp_limit(limit, 10, JOBS_MAX)
+    async with db_session() as db:
+        if job.strip():
+            row = await db.get(HostMaintenanceJob, job.strip())
+            if row is None:
+                raise ValueError(f"no maintenance job {job.strip()!r}; "
+                                 "get_maintenance_job(host=...) lists a host's jobs.")
+            jobs = [row]
+        else:
+            h = pick(host, (await db.execute(select(Host))).scalars(),
+                     kind="host", list_tool="list_hosts")
+            jobs = list((await db.execute(
+                select(HostMaintenanceJob).where(HostMaintenanceJob.host_id == h.id)
+                .order_by(HostMaintenanceJob.state.in_(maintenance.ACTIVE).desc(),
+                          HostMaintenanceJob.created_at.desc())
+                .limit(limit + 1))).scalars())
+        names = await _names(db, [j.host_id for j in jobs] + [j.drain_host_id for j in jobs]
+                             + [j.created_by for j in jobs])
+    rows = [_job_row(j, names, now) for j in jobs[:limit]]
+    return {"jobs": rows, "total": len(rows), "truncated": len(jobs) > limit}
+
+
+async def get_host_senders(host: str, range: str = "1h", direction: str = "tx", limit: int = 10) -> dict:
+    """Who sent (or received) a host's network traffic: its containers and NFS
+    mounts, ranked against the host's own total.
+
+    host: name or id (list_hosts). range: "1h" (default), "6h", "24h" or "7d"
+    (at most the raw window; the rollup keeps no per-sender detail).
+    direction: "tx" (default, what the host sent) or "rx" (what it received).
+    limit: senders listed (default 10, max 50).
+
+    The window is cut into about 120 buckets (`bucket_s` wide). `host` is the
+    host's physical interfaces: average, peak and the bucket it peaked in.
+    Each sender is a container (a swarm service's tasks added up, by service
+    name) or an NFS mount, with its average over the window, its own peak
+    and when, and `at_host_peak_bps`, what it sent in the host's peak bucket;
+    biggest there first. `unaccounted_*` is what the host carried beyond them
+    all: the host itself (a download, a backup run on the host, swarm ingress
+    passing traffic to another node). A container's traffic to another on
+    the same host counts for it without crossing an interface, so senders
+    can add up to more than the host. Per-container rates need Steward after
+    #6115; NFS bytes need agent 1.25.0. Bytes a second throughout.
+    """
+    since, key = lookback(range)
+    limit = clamp_limit(limit, 10, 50)
+    async with db_session() as db:
+        h = pick(host, (await db.execute(select(Host))).scalars(), kind="host", list_tool="list_hosts")
+        got = await senders.senders(db, h.name, since, direction=direction, limit=limit)
+    if got["host"] is None:
+        raise ValueError(f"no net_{direction}_bps recorded for host {h.name!r} in the last {key}; "
+                         "is its agent reporting? list_agents says.")
+    return {
+        "host_id": h.id, "host_name": h.name, "range": key, "direction": direction,
+        "since": iso(got["since"]), "bucket_s": got["bucket_s"],
+        "host": {"avg_bps": rnd(got["host"]["avg_bps"], 1), "peak_bps": rnd(got["host"]["peak_bps"], 1),
+                 "peak_at": iso(got["host"]["peak_at"])},
+        "senders": [{"kind": r["kind"], "name": r["name"], "at_host_peak_bps": rnd(r["at_host_peak_bps"], 1),
+                     "avg_bps": rnd(r["avg_bps"], 1), "peak_bps": rnd(r["peak_bps"], 1),
+                     "peak_at": iso(r["peak_at"])} for r in got["senders"]],
+        "more": got["more"],
+        "unaccounted_avg_bps": rnd(got["unaccounted_avg_bps"], 1),
+        "unaccounted_at_peak_bps": rnd(got["unaccounted_at_peak_bps"], 1),
+    }
+
+
+async def run_maintenance(host: str, reboot: str = "if_required") -> dict:
+    """Update a host now, as the host page's Update now does: the job starts
+    with the agent's next post and get_maintenance_job follows it.
+
+    host: the host's name or id.
+    reboot: "if_required" (default) reboots only when the update says the
+      host needs it, draining its swarm node first and making it active after;
+      "never" updates and leaves any reboot for later.
+
+    Refused while the host already has a job running, or when its agent has
+    no maintenance helper yet (Update agent installs it). Admin only, as on
+    the page. Recorded in the audit log.
+    """
+    if reboot not in ("if_required", "never"):
+        raise ValueError('reboot must be "if_required" or "never". Nothing was started.')
+    caller = current_caller()
+    async with db_session() as db, db.begin():
+        h = pick(host, (await db.execute(select(Host))).scalars(),
+                 kind="host", list_tool="list_hosts")
+        reg = (await db.execute(select(HostAgentRegistration).where(
+            HostAgentRegistration.host_id == h.id))).scalar_one_or_none()
+        try:
+            created = await maintenance.create_job(db, h, reg, caller.user_id, reboot)
+        except maintenance.CannotStart as exc:
+            raise ValueError(f"{h.name} can't be updated now: {exc}. Nothing was started.") from None
+        job_id, host_id, name = created.id, h.id, h.name
+    await audit_mcp("host_agent.maintenance", "host", host_id,
+                    {"job_id": job_id, "reboot_policy": reboot})
+    return {"job_id": job_id, "host": name, "state": "queued", "reboot_policy": reboot,
+            "next": f'get_maintenance_job(job="{job_id}") follows it; it starts with the '
+                    "agent's next post."}
+
+
+async def _failed_jobs(db, now) -> list:
+    """Each host's latest finished job, when it failed or timed out within
+    FAILED_JOB_DAYS and still flags the host (a later success clears it, and
+    so does a late return once the host is back, #6175)."""
+    rows = (await db.execute(
+        select(HostMaintenanceJob, Host.name).join(Host, Host.id == HostMaintenanceJob.host_id)
+        .where(HostMaintenanceJob.state.in_(maintenance.FINISHED),
+               HostMaintenanceJob.finished_at >= now - timedelta(days=FAILED_JOB_DAYS))
+        .order_by(HostMaintenanceJob.finished_at.desc()))).all()
+    latest = {}
+    for j, name in rows:
+        latest.setdefault(j.host_id, (j, name))
+    return [(j, name) for j, name in latest.values() if maintenance.still_failed(j)]
+
+
 async def mcp_overview(db) -> list[dict]:
-    """Agents registered more than PENDING_GRACE ago that have never checked in.
+    """Agents registered more than PENDING_GRACE ago that have never checked
+    in, and hosts whose latest maintenance job failed or timed out.
 
     Agents that stopped reporting are already flagged by the core overview
     from their metrics, so they are not repeated here.
     """
     now = now_utc()
-    return [
+    out = [
         finding("warning", "agent_never_reported", h.name,
                 "host agent was registered but has never checked in; the install may have failed",
                 since=reg.created_at, ids={"host_id": h.id}, next="list_agents()")
         for reg, h in await _agents(db)
         if reg.last_seen_at is None and now - reg.created_at > PENDING_GRACE
     ]
+    out += [
+        finding("warning", "maintenance_failed", name,
+                f"maintenance {maintenance.LABELS[j.state].lower()}"
+                + (f": {j.detail}" if j.detail else ""),
+                since=j.finished_at, ids={"host_id": j.host_id, "job_id": j.id},
+                next=f'get_maintenance_job(job="{j.id}")')
+        for j, name in await _failed_jobs(db, now)
+    ]
+    return out
 
 
 def register_tools(register) -> None:
     register(list_agents, role=UserRole.viewer, read_only=True)
+    register(list_maintenance_plans, role=UserRole.viewer, read_only=True)
+    register(get_maintenance_job, role=UserRole.viewer, read_only=True)
+    register(get_host_senders, role=UserRole.viewer, read_only=True)
+    register(run_maintenance, role=UserRole.admin, read_only=False)

plugins/host_agent/metrics_query.py

@@ -21,10 +21,20 @@ SOURCE_MODULE = "host_agent"
 HISTORY_METRICS = (
     "cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m",
     "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
-    "temp_c_max", "psi_mem_some_avg10",
+    "temp_c_max", "psi_mem_some_avg10", "nfs_exe_max_ms", "mounts_stalled",
 )
 
 
+async def has_nfs(session, host_name: str) -> bool:
+    """Whether the host's agent has reported an NFS mount's delay (#5996)."""
+    from steward.models.metrics import PluginMetricLatest
+    return (await session.execute(
+        select(PluginMetricLatest.metric_name).where(
+            PluginMetricLatest.source_module == SOURCE_MODULE,
+            PluginMetricLatest.resource_name.startswith(host_name + ":"),
+            PluginMetricLatest.metric_name == "nfs_ops_per_sec").limit(1))).first() is not None
+
+
 async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]:
     """{resource_name: {metric: value}} — latest sample per (resource, metric) for
     a host + its sub-resources, from plugin_metric_latest (#5208). A host whose

plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py

@@ -0,0 +1,24 @@
+# plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py
+"""host_agent: how far each host's clock is from Steward's (milestone 490)
+
+Revision ID: host_agent_003_clock_offset
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_003_clock_offset"
+down_revision: Union[str, None] = "host_agent_002_host_ip"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("host_agent_registrations", sa.Column("clock_offset_s", sa.Float(), nullable=True))
+    op.add_column("host_agent_registrations",
+                  sa.Column("clock_checked_at", sa.DateTime(timezone=True), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("host_agent_registrations", "clock_checked_at")
+    op.drop_column("host_agent_registrations", "clock_offset_s")

plugins/host_agent/migrations/versions/host_agent_004_downtime.py

@@ -0,0 +1,32 @@
+# plugins/host_agent/migrations/versions/host_agent_004_downtime.py
+"""host_agent: each host's downtime and what it was (#5756)
+
+Revision ID: host_agent_004_downtime
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_004_downtime"
+down_revision: Union[str, None] = "host_agent_003_clock_offset"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "host_downtime",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("host_id", sa.String(36), sa.ForeignKey("hosts.id", ondelete="CASCADE"),
+                  nullable=False),
+        sa.Column("kind", sa.String(16), nullable=False),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+    )
+    op.create_index("ix_host_downtime_host_id", "host_downtime", ["host_id"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_host_downtime_host_id", table_name="host_downtime")
+    op.drop_table("host_downtime")

plugins/host_agent/migrations/versions/host_agent_005_updates.py

@@ -0,0 +1,34 @@
+# plugins/host_agent/migrations/versions/host_agent_005_updates.py
+"""host_agent: a host's pending updates and whether it needs a reboot (#5765)
+
+Revision ID: host_agent_005_updates
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_005_updates"
+down_revision: Union[str, None] = "host_agent_004_downtime"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+_COLUMNS = (
+    ("updates_manager", sa.String(8)),
+    ("updates_pending", sa.Integer()),
+    ("updates_security", sa.Integer()),
+    ("updates_checked_at", sa.DateTime(timezone=True)),
+    ("updates_error", sa.String(200)),
+    ("reboot_required", sa.Boolean()),
+    ("reboot_required_since", sa.DateTime(timezone=True)),
+    ("reboot_packages", sa.Text()),
+)
+
+
+def upgrade() -> None:
+    for name, kind in _COLUMNS:
+        op.add_column("host_agent_registrations", sa.Column(name, kind, nullable=True))
+
+
+def downgrade() -> None:
+    for name, _kind in reversed(_COLUMNS):
+        op.drop_column("host_agent_registrations", name)

plugins/host_agent/migrations/versions/host_agent_006_maint_helper.py

@@ -0,0 +1,21 @@
+# plugins/host_agent/migrations/versions/host_agent_006_maint_helper.py
+"""host_agent: whether a host has the maintenance helper (#5766)
+
+Revision ID: host_agent_006_maint_helper
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_006_maint_helper"
+down_revision: Union[str, None] = "host_agent_005_updates"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("host_agent_registrations", sa.Column("maint_helper", sa.Boolean(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("host_agent_registrations", "maint_helper")

plugins/host_agent/migrations/versions/host_agent_007_maint_jobs.py

@@ -0,0 +1,47 @@
+"""host_agent: a host's maintenance job, followed through its reboot (#5767)
+
+Revision ID: host_agent_007_maint_jobs
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_007_maint_jobs"
+down_revision: Union[str, None] = "host_agent_006_maint_helper"
+branch_labels: Union[str, Sequence[str], None] = None
+# maintenance_windows (core 0010) must exist before the window_id key.
+depends_on: Union[str, Sequence[str], None] = ("0010_maintenance_windows",)
+
+
+def upgrade() -> None:
+    op.create_table(
+        "host_maintenance_jobs",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("host_id", sa.String(36), sa.ForeignKey("hosts.id", ondelete="CASCADE"),
+                  nullable=False),
+        sa.Column("state", sa.String(16), nullable=False),
+        sa.Column("state_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("reboot_policy", sa.String(16), nullable=False),
+        sa.Column("step_id", sa.String(64), nullable=True),
+        sa.Column("step_seen", sa.Boolean(), nullable=False, server_default=sa.false()),
+        sa.Column("node_id", sa.String(64), nullable=True),
+        sa.Column("drain_host_id", sa.String(36), sa.ForeignKey("hosts.id", ondelete="SET NULL"),
+                  nullable=True),
+        sa.Column("packages", sa.Integer(), nullable=True),
+        sa.Column("reboot_needed", sa.Boolean(), nullable=True),
+        sa.Column("back_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("detail", sa.Text(), nullable=True),
+        sa.Column("window_id", sa.String(36),
+                  sa.ForeignKey("maintenance_windows.id", ondelete="SET NULL"), nullable=True),
+        sa.Column("created_by", sa.String(36), sa.ForeignKey("users.id", ondelete="SET NULL"),
+                  nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.create_index("ix_host_maintenance_jobs_host_id", "host_maintenance_jobs", ["host_id"])
+    op.create_index("ix_host_maintenance_jobs_state", "host_maintenance_jobs", ["state"])
+    op.create_index("ix_host_maintenance_jobs_step_id", "host_maintenance_jobs", ["step_id"])
+
+
+def downgrade() -> None:
+    op.drop_table("host_maintenance_jobs")

plugins/host_agent/migrations/versions/host_agent_008_maint_plans.py

@@ -0,0 +1,61 @@
+"""host_agent: a group's maintenance cadence and its runs (#5768)
+
+Revision ID: host_agent_008_maint_plans
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_008_maint_plans"
+down_revision: Union[str, None] = "host_agent_007_maint_jobs"
+branch_labels: Union[str, Sequence[str], None] = None
+# ansible_groups (core 0016) must exist before the group_id key.
+depends_on: Union[str, Sequence[str], None] = ("0016_ansible_inventory_groups",)
+
+
+def upgrade() -> None:
+    op.create_table(
+        "host_maintenance_plans",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("name", sa.String(128), nullable=False),
+        sa.Column("group_id", sa.String(36), sa.ForeignKey("ansible_groups.id", ondelete="CASCADE"),
+                  nullable=False),
+        sa.Column("weekday", sa.Integer(), nullable=False),
+        sa.Column("at_time", sa.String(5), nullable=False),
+        sa.Column("time_zone", sa.String(64), nullable=False, server_default="UTC"),
+        sa.Column("window_minutes", sa.Integer(), nullable=False, server_default="240"),
+        sa.Column("reboot_policy", sa.String(16), nullable=False, server_default="if_required"),
+        sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column("last_slot_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_by", sa.String(36), sa.ForeignKey("users.id", ondelete="SET NULL"),
+                  nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+    )
+    op.create_table(
+        "host_maintenance_runs",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("plan_id", sa.String(36),
+                  sa.ForeignKey("host_maintenance_plans.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("state", sa.String(16), nullable=False),
+        sa.Column("host_ids", sa.Text(), nullable=False, server_default="[]"),
+        sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("skipped", sa.Text(), nullable=False, server_default="[]"),
+        sa.Column("settle_since", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("detail", sa.Text(), nullable=True),
+        sa.Column("manual", sa.Boolean(), nullable=False, server_default=sa.false()),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.create_index("ix_host_maintenance_runs_plan_id", "host_maintenance_runs", ["plan_id"])
+    op.create_index("ix_host_maintenance_runs_state", "host_maintenance_runs", ["state"])
+    op.add_column("host_maintenance_jobs", sa.Column(
+        "run_id", sa.String(36), sa.ForeignKey("host_maintenance_runs.id", ondelete="SET NULL"),
+        nullable=True))
+    op.create_index("ix_host_maintenance_jobs_run_id", "host_maintenance_jobs", ["run_id"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_host_maintenance_jobs_run_id", "host_maintenance_jobs")
+    op.drop_column("host_maintenance_jobs", "run_id")
+    op.drop_table("host_maintenance_runs")
+    op.drop_table("host_maintenance_plans")

plugins/host_agent/migrations/versions/host_agent_009_drive_health.py

@@ -0,0 +1,24 @@
+# plugins/host_agent/migrations/versions/host_agent_009_drive_health.py
+"""host_agent: each host's newest drive-health report (#5789)
+
+Revision ID: host_agent_009_drive_health
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_009_drive_health"
+down_revision: Union[str, None] = "host_agent_008_maint_plans"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("host_agent_registrations", sa.Column("drive_health", sa.JSON(), nullable=True))
+    op.add_column("host_agent_registrations",
+                  sa.Column("drive_health_at", sa.DateTime(timezone=True), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("host_agent_registrations", "drive_health_at")
+    op.drop_column("host_agent_registrations", "drive_health")

plugins/host_agent/migrations/versions/host_agent_010_maint_check.py

@@ -0,0 +1,32 @@
+# plugins/host_agent/migrations/versions/host_agent_010_maint_check.py
+"""host_agent: a maintenance job's check of what its update would change, and
+when it drained its node (#6098)
+
+Revision ID: host_agent_010_maint_check
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_010_maint_check"
+down_revision: Union[str, None] = "host_agent_009_drive_health"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("host_maintenance_jobs", sa.Column("docker_update", sa.Boolean(), nullable=True))
+    op.add_column("host_maintenance_jobs", sa.Column("checked", sa.JSON(), nullable=True))
+    op.add_column("host_maintenance_jobs",
+                  sa.Column("drained_at", sa.DateTime(timezone=True), nullable=True))
+    # A job already past its drain when this lands still makes its node
+    # active again: before this, every job that drained went on to undrain.
+    op.execute("UPDATE host_maintenance_jobs SET drained_at = state_at "
+               "WHERE state IN ('rebooting', 'undraining') "
+               "AND node_id IS NOT NULL AND drain_host_id IS NOT NULL")
+
+
+def downgrade() -> None:
+    op.drop_column("host_maintenance_jobs", "drained_at")
+    op.drop_column("host_maintenance_jobs", "checked")
+    op.drop_column("host_maintenance_jobs", "docker_update")

plugins/host_agent/migrations/versions/host_agent_011_came_back.py

@@ -0,0 +1,23 @@
+# plugins/host_agent/migrations/versions/host_agent_011_came_back.py
+"""host_agent: when a job failed for not coming back into service in time
+first found its host back (#6175)
+
+Revision ID: host_agent_011_came_back
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_011_came_back"
+down_revision: Union[str, None] = "host_agent_010_maint_check"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("host_maintenance_jobs",
+                  sa.Column("came_back_at", sa.DateTime(timezone=True), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("host_maintenance_jobs", "came_back_at")

plugins/host_agent/migrations/versions/host_agent_012_ends_as.py

@@ -0,0 +1,22 @@
+# plugins/host_agent/migrations/versions/host_agent_012_ends_as.py
+"""host_agent: what a job given up while its node may be drained finishes as,
+once the node is active again
+
+Revision ID: host_agent_012_ends_as
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_012_ends_as"
+down_revision: Union[str, None] = "host_agent_011_came_back"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("host_maintenance_jobs", sa.Column("ends_as", sa.String(16), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("host_maintenance_jobs", "ends_as")

plugins/host_agent/migrations/versions/host_agent_013_spread.py

@@ -0,0 +1,28 @@
+# plugins/host_agent/migrations/versions/host_agent_013_spread.py
+"""host_agent: a maintenance run spreads swarm services back out after its
+last host, and a plan can keep named services where they are
+
+Revision ID: host_agent_013_spread
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_013_spread"
+down_revision: Union[str, None] = "host_agent_012_ends_as"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("host_maintenance_plans", sa.Column("spread", sa.Boolean, nullable=False,
+                                                      server_default=sa.true()))
+    op.add_column("host_maintenance_plans", sa.Column("keep_in_place", sa.Text, nullable=False,
+                                                      server_default=""))
+    op.add_column("host_maintenance_runs", sa.Column("rebalance", sa.JSON, nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("host_maintenance_runs", "rebalance")
+    op.drop_column("host_maintenance_plans", "keep_in_place")
+    op.drop_column("host_maintenance_plans", "spread")

plugins/host_agent/models.py

@@ -2,7 +2,7 @@
 from __future__ import annotations
 import uuid
 from datetime import datetime, timezone
-from sqlalchemy import Column, String, DateTime, ForeignKey
+from sqlalchemy import JSON, Boolean, Column, String, DateTime, Float, ForeignKey, Integer, Text
 from steward.models.base import Base
 
 
@@ -29,6 +29,155 @@ class HostAgentRegistration(Base):
     # Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix).
     host_ip = Column(String(45), nullable=True)
     last_seen_at = Column(DateTime(timezone=True), nullable=True)
+    # How far this host's clock is from Steward's, in seconds, positive when
+    # the host is ahead, measured from the agent's sent_at on the newest post;
+    # and when that was. None until an agent that stamps its posts reports.
+    clock_offset_s = Column(Float, nullable=True)
+    clock_checked_at = Column(DateTime(timezone=True), nullable=True)
+    # What the package manager has waiting, from the agent's hourly check
+    # (agent ≥ 1.16.0, #5765). updates_security is None when the manager
+    # can't say; updates_error is why the last check failed.
+    updates_manager = Column(String(8), nullable=True)
+    updates_pending = Column(Integer, nullable=True)
+    updates_security = Column(Integer, nullable=True)
+    updates_checked_at = Column(DateTime(timezone=True), nullable=True)
+    updates_error = Column(String(200), nullable=True)
+    # Whether the host needs a reboot (None: it can't say), since when, and
+    # the packages that asked for it, comma-separated.
+    reboot_required = Column(Boolean, nullable=True)
+    reboot_required_since = Column(DateTime(timezone=True), nullable=True)
+    reboot_packages = Column(Text, nullable=True)
+    # Whether the host has the root maintenance helper (steward-maint.path and
+    # its request directory) that updates and reboots go through (#5766); None
+    # until an agent that says reports.
+    maint_helper = Column(Boolean, nullable=True)
+    # The host's newest drive-health report, as the root helper read it with
+    # smartctl (agent ≥ 1.20.0, #5789): {"checked_at", "smartctl", "drives"} or
+    # {"error"}. drives.py judges it; the figures are also metrics.
+    drive_health = Column(JSON, nullable=True)
+    drive_health_at = Column(DateTime(timezone=True), nullable=True)
     created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
     updated_at = Column(DateTime(timezone=True), nullable=False,
                         default=_utcnow, onupdate=_utcnow)
+
+
+class HostDowntime(Base):
+    """A span a host's agent was away, and what the agent said it was (#5756).
+
+    kind is "reboot" (the host went down and came back), "agent_stop" (the
+    agent stopped; the host stayed up), "unclean" (the host came back from a
+    boot nobody announced: power loss or a crash) or "lost_contact" (the agent
+    ran but couldn't reach Steward). ended_at is None while it is going on.
+    """
+    __tablename__ = "host_downtime"
+
+    id = Column(String(36), primary_key=True, default=_uuid)
+    host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
+                     nullable=False, index=True)
+    kind = Column(String(16), nullable=False)
+    started_at = Column(DateTime(timezone=True), nullable=False)
+    ended_at = Column(DateTime(timezone=True), nullable=True)
+    created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
+
+
+class HostMaintenanceJob(Base):
+    """One host's update, and its reboot when the update needs one (#5767).
+
+    Steward keeps the job here and the host's agent carries out each step
+    through its root helper; maintenance.py has the states and transitions.
+    step_id is the helper request id of the current step, and step_seen says a
+    result for it has arrived, so it stops being handed out. node_id and
+    drain_host_id are set when the host is a swarm node: the node to drain and
+    the manager whose agent drains it.
+    """
+    __tablename__ = "host_maintenance_jobs"
+
+    id = Column(String(36), primary_key=True, default=_uuid)
+    host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
+                     nullable=False, index=True)
+    state = Column(String(16), nullable=False, default="queued", index=True)
+    state_at = Column(DateTime(timezone=True), nullable=True)
+    reboot_policy = Column(String(16), nullable=False, default="if_required")
+    step_id = Column(String(64), nullable=True, index=True)
+    step_seen = Column(Boolean, nullable=False, default=False)
+    node_id = Column(String(64), nullable=True)
+    drain_host_id = Column(String(36), ForeignKey("hosts.id", ondelete="SET NULL"), nullable=True)
+    packages = Column(Integer, nullable=True)
+    reboot_needed = Column(Boolean, nullable=True)
+    back_at = Column(DateTime(timezone=True), nullable=True)
+    detail = Column(Text, nullable=True)
+    window_id = Column(String(36), ForeignKey("maintenance_windows.id", ondelete="SET NULL"),
+                       nullable=True)
+    created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
+    created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
+    finished_at = Column(DateTime(timezone=True), nullable=True)
+    # The cadence run this job belongs to (#5768); None for Update now.
+    run_id = Column(String(36), ForeignKey("host_maintenance_runs.id", ondelete="SET NULL"),
+                    nullable=True, index=True)
+    # What the check said the update would change (#6098): whether it
+    # upgrades Docker, and {pending, docker_packages, containers} with the
+    # containers then running outside the swarm. drained_at is when the job's
+    # drain finished, before the update or before the reboot.
+    docker_update = Column(Boolean, nullable=True)
+    checked = Column(JSON, nullable=True)
+    drained_at = Column(DateTime(timezone=True), nullable=True)
+    # A job failed for not coming back into service in time: when its host
+    # was first found back (#6175). From then on the job no longer flags it.
+    came_back_at = Column(DateTime(timezone=True), nullable=True)
+    # A job given up (failed or timed_out) while its node may be drained makes
+    # the node active again first; ends_as is what it finishes as after that.
+    ends_as = Column(String(16), nullable=True)
+
+
+class HostMaintenancePlan(Base):
+    """A group's maintenance cadence (#5768): every week on `weekday` at
+    `at_time` in `time_zone`, its hosts are updated one at a time, each
+    rebooting per `reboot_policy`. No host is started after `window_minutes`
+    from the run's start. last_slot_at is the last scheduled slot that fired;
+    Run now leaves it alone, so the schedule keeps its time. With `spread`, a
+    run that drained swarm nodes ends by spreading services back out, leaving
+    the services named in `keep_in_place` (comma-separated) where they are."""
+    __tablename__ = "host_maintenance_plans"
+
+    id = Column(String(36), primary_key=True, default=_uuid)
+    name = Column(String(128), nullable=False)
+    group_id = Column(String(36), ForeignKey("ansible_groups.id", ondelete="CASCADE"),
+                      nullable=False)
+    weekday = Column(Integer, nullable=False)              # 0 = Monday
+    at_time = Column(String(5), nullable=False)            # "HH:MM"
+    time_zone = Column(String(64), nullable=False, default="UTC")
+    window_minutes = Column(Integer, nullable=False, default=240)
+    reboot_policy = Column(String(16), nullable=False, default="if_required")
+    enabled = Column(Boolean, nullable=False, default=True)
+    spread = Column(Boolean, nullable=False, default=True)
+    keep_in_place = Column(Text, nullable=False, default="")
+    last_slot_at = Column(DateTime(timezone=True), nullable=True)
+    created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
+    created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
+
+
+class HostMaintenanceRun(Base):
+    """One run of a plan: its hosts in order, one at a time (#5768).
+
+    host_ids is the order fixed at the start (JSON list); position is the host
+    being worked on; skipped is [{"host_id", "reason"}] for hosts the run
+    passed over. settle_since is when the current host's job finished and the
+    run began waiting for the host to be back in service. rebalance is the
+    spreading of swarm services back out once the last host is back
+    (rebalance.py): its state, the manager carrying it out, what it moved and
+    what it left and why; None while not reached or not wanted."""
+    __tablename__ = "host_maintenance_runs"
+
+    id = Column(String(36), primary_key=True, default=_uuid)
+    plan_id = Column(String(36), ForeignKey("host_maintenance_plans.id", ondelete="CASCADE"),
+                     nullable=False, index=True)
+    state = Column(String(16), nullable=False, default="running", index=True)
+    host_ids = Column(Text, nullable=False, default="[]")
+    position = Column(Integer, nullable=False, default=0)
+    skipped = Column(Text, nullable=False, default="[]")
+    settle_since = Column(DateTime(timezone=True), nullable=True)
+    detail = Column(Text, nullable=True)
+    manual = Column(Boolean, nullable=False, default=False)
+    rebalance = Column(JSON, nullable=True)
+    started_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
+    finished_at = Column(DateTime(timezone=True), nullable=True)

plugins/host_agent/plugin.yaml

@@ -16,4 +16,7 @@ tags:
 
 config:
   stale_after_seconds: 180
+  # How long a reboot or restart the agent announced explains its silence
+  # before "agent offline" fires (agent >= 1.14.0, #5756).
+  reboot_grace_seconds: 600
   default_interval_seconds: 30

plugins/host_agent/rebalance.py

@@ -0,0 +1,374 @@
+# plugins/host_agent/rebalance.py
+"""A maintenance run spreads swarm services back out after its last host.
+
+A run drains each swarm node in turn, and the swarm never moves a task back
+on its own, so every run used to end with the services on whichever node was
+drained first. Once the run's last host is back in service, Steward picks
+services to move from the fullest node to the emptiest and hands the list to
+a swarm manager's helper (maint.py `rebalance`), which force-updates each in
+turn: spec unchanged, no image pulled, and the swarm starts it again on its
+emptiest node.
+
+A service moves only when the emptier node already holds the exact image the
+service is pinned to (its agent reports every image digest it holds), so a
+pruned image or a digest the registry has since deleted never strands one.
+A service the plan keeps in place, one below its replicas, and one a
+constraint places are left where they are, and the run says why.
+
+The choice is model-free (`choose`); the database side is below it, driven by
+cadence.advance, the agent's posts (handoffs, results) and the minute tick.
+"""
+from __future__ import annotations
+
+import json
+from datetime import datetime, timedelta
+
+from . import maintenance
+
+# The first agent whose helper can rebalance, and that reports image digests.
+REBALANCE_AGENT = (1, 26, 0)
+# How many services one rebalance moves at most; the helper refuses more.
+SERVICES_MAX = 48
+# A node's list of images older than this isn't trusted (agents report it
+# every 10 minutes).
+DIGESTS_FRESH = timedelta(minutes=30)
+# The helper starts no move after 40 min and gives each 5 to settle.
+DEADLINE = timedelta(minutes=60)
+STATES = ("handed", "done", "failed", "timed_out", "nothing")
+WINDOW_NAME = "Maintenance: spreading swarm services back out"
+
+
+def keep_list(text: str | None) -> list[str]:
+    """A plan's keep_in_place as names, in the order written."""
+    out = []
+    for name in (text or "").replace("\n", ",").split(","):
+        name = name.strip()
+        if name and name not in out:
+            out.append(name)
+    return out
+
+
+def _repo(image: str) -> str:
+    """An image reference's repository, without tag or digest, in the short
+    form Docker reports (no docker.io/ or library/)."""
+    ref = image.split("@", 1)[0]
+    slash = ref.rfind("/")
+    colon = ref.rfind(":")
+    if colon > slash:
+        ref = ref[:colon]
+    for prefix in ("docker.io/", "index.docker.io/"):
+        if ref.startswith(prefix):
+            ref = ref[len(prefix):]
+    if ref.startswith("library/"):
+        ref = ref[len("library/"):]
+    return ref
+
+
+def pinned(image: str | None, digest: str | None) -> str | None:
+    """"repo@sha256:…" for a service's image, or None when it isn't pinned."""
+    if not image or not digest or not digest.startswith("sha256:"):
+        return None
+    return f"{_repo(image)}@{digest}"
+
+
+def held(digests) -> set:
+    """A node's reported digests, in the same form as pinned()."""
+    return {f"{_repo(d)}@{d.split('@', 1)[1]}" for d in digests or []
+            if isinstance(d, str) and "@sha256:" in d}
+
+
+def choose(services: list, nodes: dict, images: dict, keep=(),
+           limit: int = SERVICES_MAX) -> tuple[list, list]:
+    """(services to move, [{"service", "reason"}] left where they are).
+
+    services: {"name", "mode", "desired", "running", "image" ("repo@sha256:…"
+    or None), "placement": [{"node_id", "running"}]}; nodes: {node id: name}
+    for the nodes to balance across (ready and active); images: {node id: set
+    of "repo@sha256:…"}, a node missing when its list is unknown.
+
+    While the fullest node has two or more tasks than the emptiest, a service
+    whose tasks all run on the fullest moves; one with several replicas there
+    first, as one node holding them all defeats the replicas. A move counts
+    as its share of the replicas going to the emptiest. What is reported left
+    is what kept the nodes from evening out: the services that would have
+    moved next but can't."""
+    if len(nodes) < 2:
+        return [], []
+    counts = {n: 0 for n in nodes}
+    for s in services:
+        for p in s.get("placement") or []:
+            if p.get("node_id") in counts:
+                counts[p["node_id"]] += int(p.get("running") or 0)
+    keep = set(keep)
+    order = sorted(services, key=lambda s: (-int(s.get("running") or 0), s["name"]))
+    moves: list = []
+    left: dict = {}
+    while len(moves) < limit:
+        src = max(counts, key=lambda n: (counts[n], n))
+        dst = min(counts, key=lambda n: (counts[n], n))
+        gap = counts[src] - counts[dst]
+        if gap < 2:
+            break
+        pick, blocked = None, {}
+        for s in order:
+            name, running = s["name"], int(s.get("running") or 0)
+            on = {p.get("node_id"): int(p.get("running") or 0) for p in s.get("placement") or []
+                  if int(p.get("running") or 0) > 0}
+            if name in moves or s.get("mode") != "replicated" or set(on) != {src}:
+                continue
+            share = max(1, running // len(nodes))
+            if 2 * share > gap:
+                continue                    # it would only tip the balance the other way
+            if name in keep:
+                blocked[name] = "kept in place by the plan"
+            elif running < int(s.get("desired") or 0):
+                blocked[name] = "below its replicas"
+            elif not s.get("image"):
+                blocked[name] = "its image isn't pinned to a digest"
+            elif dst not in images:
+                blocked[name] = f"no image list from {nodes[dst]} (its agent needs 1.26.0)"
+            elif s["image"] not in images[dst]:
+                blocked[name] = f"its image isn't on {nodes[dst]}"
+            else:
+                pick = s
+                break
+        if pick is None:
+            left = blocked                  # what kept the nodes from evening out
+            break
+        share = max(1, int(pick.get("running") or 0) // len(nodes))
+        moves.append(pick["name"])
+        counts[src] -= share
+        counts[dst] += share
+    return moves, [{"service": n, "reason": r} for n, r in sorted(left.items())]
+
+
+def can_rebalance(agent_version) -> bool:
+    try:
+        return tuple(int(x) for x in str(agent_version).split(".")[:3]) >= REBALANCE_AGENT
+    except ValueError:
+        return False
+
+
+def step_id(run) -> str:
+    return f"{run.id}-rebalance"
+
+
+def _set(run, **changes) -> None:
+    """Change the run's rebalance record (a new dict, so the JSON column
+    sees the change)."""
+    run.rebalance = {**(run.rebalance or {}), **changes}
+
+
+def handoff(run) -> dict | None:
+    """The step to hand the manager's agent, until its result arrives."""
+    rb = run.rebalance or {}
+    if rb.get("state") != "handed" or rb.get("seen"):
+        return None
+    return {"id": rb["step_id"], "action": "rebalance", "services": rb.get("planned") or []}
+
+
+def apply_result(run, event: dict, now: datetime) -> bool:
+    """A `maintenance` event for the run's rebalance; True when it changed."""
+    rb = run.rebalance or {}
+    if rb.get("state") != "handed" or event.get("id") != rb.get("step_id"):
+        return False
+    state = event.get("state")
+    if state == "running":
+        if rb.get("seen"):
+            return False
+        _set(run, seen=True)
+        return True
+    if state not in ("done", "failed"):
+        return False
+    moved = [str(m)[:64] for m in event.get("moved") or [] if isinstance(m, str)][:SERVICES_MAX]
+    gone = [{"service": str(x.get("service"))[:64], "reason": str(x.get("reason"))[:160]}
+            for x in event.get("left") or [] if isinstance(x, dict)][:SERVICES_MAX]
+    ok = state == "done" and event.get("ok") is not False
+    _set(run, state="done" if ok else "failed", moved=moved, left=(rb.get("left") or []) + gone,
+         detail=str(event.get("detail") or "")[:maintenance.DETAIL_MAX] or None,
+         finished_at=now.isoformat())
+    return True
+
+
+def check_deadline(run, now: datetime) -> bool:
+    rb = run.rebalance or {}
+    if rb.get("state") != "handed" or now - datetime.fromisoformat(rb["at"]) <= DEADLINE:
+        return False
+    _set(run, state="timed_out", finished_at=now.isoformat(),
+         detail=f"the manager didn't finish within {int(DEADLINE.total_seconds() // 60)} min")
+    return True
+
+
+def outcome(run) -> tuple[str, str | None]:
+    """How a run whose hosts are all done ends: ("done" | "failed", detail)."""
+    rb = run.rebalance or {}
+    state = rb.get("state")
+    left = rb.get("left") or []
+    tail = f"; {len(left)} left where they are" if left else ""
+    if state == "done":
+        n = len(rb.get("moved") or [])
+        return "done", f"spread {n} swarm service{'s' if n != 1 else ''} back out{tail}"
+    if state in ("failed", "timed_out"):
+        return "failed", f"spreading swarm services back out: {rb.get('detail') or state}"
+    if state == "nothing" and rb.get("detail"):
+        return "done", f"swarm services not spread back out: {rb['detail']}"
+    if state == "nothing" and left:
+        return "done", f"no swarm service could be spread back out{tail}"
+    return "done", None
+
+
+def label(run) -> tuple[str, bool, bool] | None:
+    """(text, bad, busy) for the run's swarm row on the page, or None when
+    the run has no rebalance."""
+    rb = run.rebalance or {}
+    state = rb.get("state")
+    if state is None:
+        return None
+    if state == "handed":
+        n = len(rb.get("planned") or [])
+        return f"Spreading {n} service{'s' if n != 1 else ''} back out", False, True
+    text = outcome(run)[1] or "Nothing to spread back out"
+    return text[:1].upper() + text[1:], state in ("failed", "timed_out"), False
+
+
+# ── applied to the database ──────────────────────────────────────────────────
+
+async def _freshest(session, model, key: str) -> list:
+    """The newest row per `key` (several managers may report the swarm)."""
+    from sqlalchemy import select
+    rows: dict = {}
+    for r in (await session.execute(select(model))).scalars():
+        k = getattr(r, key)
+        if k not in rows or r.updated_at > rows[k].updated_at:
+            rows[k] = r
+    return list(rows.values())
+
+
+async def _swarm_view(session, now: datetime) -> tuple[list, dict, dict] | None:
+    """(services, nodes, images) as choose() takes them, from what the Docker
+    plugin last stored; None without it."""
+    from sqlalchemy import select
+    from steward.models.hosts import Host
+    models = maintenance.docker_models()
+    if models is None:
+        return None
+    services = [{"name": s.service_name, "mode": s.mode, "desired": s.desired,
+                 "running": s.running, "image": pinned(s.image, s.image_digest),
+                 "placement": json.loads(s.placement_json or "[]")}
+                for s in await _freshest(session, models.DockerSwarmService, "service_name")]
+    nodes = {n.node_id: n.hostname or n.node_id
+             for n in await _freshest(session, models.DockerSwarmNode, "node_id")
+             if n.status == "ready" and n.availability == "active"}
+    hosts = dict((await session.execute(select(Host.name, Host.id))).all())
+    by_name = {name.lower(): hid for name, hid in hosts.items()}
+    images = {}
+    for node_id, name in nodes.items():
+        host_id = by_name.get(name.lower())
+        disk = await session.get(models.DockerDiskUsage, host_id) if host_id else None
+        if disk is not None and now - disk.updated_at <= DIGESTS_FRESH:
+            got = held(json.loads(disk.digests_json or "[]"))
+            if got:
+                images[node_id] = got
+    return services, nodes, images
+
+
+async def _manager(session, jobs: list) -> tuple[str | None, str | None]:
+    """(host id, None) of a swarm manager whose helper can rebalance, the one
+    that drained the run's nodes first; (None, why) when there is none."""
+    from sqlalchemy import select
+    from .models import HostAgentRegistration
+    _, managers, _ = await maintenance._swarm(session)
+    regs = {r.host_id: r for r in (await session.execute(select(HostAgentRegistration).where(
+        HostAgentRegistration.host_id.in_(list(managers)),
+        HostAgentRegistration.maint_helper.is_(True)))).scalars()} if managers else {}
+    drained_by = [j.drain_host_id for j in jobs if j.drain_host_id]
+    for host_id in drained_by + sorted(regs):
+        reg = regs.get(host_id)
+        if reg is not None and can_rebalance(reg.agent_version):
+            return host_id, None
+    if regs:
+        return None, "no swarm manager's agent can do it yet: run Update agent on a manager"
+    return None, "no swarm manager has the maintenance helper"
+
+
+async def begin(session, run, plan, now: datetime) -> bool:
+    """The run's last host is back in service: hand a manager the services to
+    move, or record why there are none. True while a rebalance is handed out."""
+    from sqlalchemy import select
+    from .models import HostMaintenanceJob
+    if not plan.spread or run.rebalance is not None:
+        return False
+    jobs = (await session.execute(select(HostMaintenanceJob).where(
+        HostMaintenanceJob.run_id == run.id))).scalars().all()
+    if not any(maintenance.drains(j) for j in jobs):
+        return False                        # nothing was drained: nothing to put back
+    view = await _swarm_view(session, now)
+    if view is None:
+        return False
+    services, nodes, images = view
+    moves, left = choose(services, nodes, images, keep_list(plan.keep_in_place))
+    if not moves:
+        run.rebalance = {"state": "nothing", "left": left, "at": now.isoformat()}
+        return False
+    host_id, why = await _manager(session, jobs)
+    if host_id is None:
+        run.rebalance = {"state": "nothing", "left": left, "detail": why, "at": now.isoformat()}
+        return False
+    run.rebalance = {"state": "handed", "step_id": step_id(run), "host_id": host_id,
+                     "at": now.isoformat(), "planned": moves, "left": left, "seen": False,
+                     "window_id": await _window(session, plan, now)}
+    return True
+
+
+async def _window(session, plan, now: datetime) -> str | None:
+    """A window over the Docker figures while services restart, so their
+    restarts raise no alert; ended when the rebalance does."""
+    from steward.models.alerts import MaintenanceWindow
+    if plan.created_by is None:
+        return None
+    w = MaintenanceWindow(name=WINDOW_NAME, start_at=now, end_at=now + DEADLINE,
+                          scope_source="docker", scope_resource=None, created_by=plan.created_by)
+    session.add(w)
+    await session.flush()
+    return w.id
+
+
+async def finish(session, run, now: datetime) -> None:
+    """The rebalance ended: its window ends, and it goes on the timeline."""
+    from steward.models.alerts import MaintenanceWindow
+    from . import timeline
+    rb = run.rebalance or {}
+    if rb.get("window_id"):
+        w = await session.get(MaintenanceWindow, rb["window_id"])
+        if w is not None:
+            w.end_at = max(w.start_at, now)
+    if rb.get("state") in ("done", "failed", "timed_out"):
+        await timeline.write(session, f"maintenance run {run.id}", timeline.run_entry(session, run))
+
+
+async def running_runs(session) -> list:
+    from sqlalchemy import select
+    from .models import HostMaintenanceRun
+    return (await session.execute(select(HostMaintenanceRun).where(
+        HostMaintenanceRun.state == "running"))).scalars().all()
+
+
+async def handoffs(session, host_id: str) -> list[dict]:
+    """The rebalance steps this host's agent should carry out now."""
+    out = []
+    for run in await running_runs(session):
+        if (run.rebalance or {}).get("host_id") == host_id:
+            step = handoff(run)
+            if step is not None:
+                out.append(step)
+    return out
+
+
+async def apply_event(session, event: dict, now: datetime) -> bool:
+    """Apply a `maintenance` event to the running run whose rebalance it is."""
+    for run in await running_runs(session):
+        if apply_result(run, event, now):
+            if (run.rebalance or {}).get("state") != "handed":
+                await finish(session, run, now)
+            return True
+    return False

plugins/host_agent/scheduler.py

@@ -8,25 +8,69 @@ from sqlalchemy import select
 
 from steward.core.alerts import MetricBatch
 from steward.core.scheduler import ScheduledTask
+from steward.core.watch import watch
+from steward.hosts.service import agent_stale_after_seconds
 from steward.models.hosts import Host
+from steward.models.metrics import PluginMetricLatest
+from .downtime import announced_stops, restart_grace_seconds
 from .models import HostAgentRegistration
 
 logger = logging.getLogger(__name__)
 
 
+def arrivals(app) -> dict[str, datetime]:
+    """{host id: when its agent's newest post reached Steward}, in memory (#5514).
+
+    last_seen_at is written in the ingest's own transaction, so when Steward
+    can't store a post (its storage stalls and the request is dropped) the
+    agent's report is lost with it, and an agent posting every few seconds read
+    as offline. A post that arrived is the agent reaching Steward; storing it
+    is Steward's part. Set before any write, so it holds however slow they are.
+    """
+    got = getattr(app, "_agent_arrivals", None)
+    if got is None:
+        got = app._agent_arrivals = {}
+    return got
+
+
 def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
-                       now: datetime) -> list[tuple[str, float]]:
+                       now: datetime, watching_since: datetime | None = None,
+                       arrived: dict[str, datetime] | None = None,
+                       already_silent: set[str] | frozenset = frozenset(),
+                       explained_until: dict[str, datetime] | None = None,
+                       ) -> list[tuple[str, float]]:
     """(host name, seconds since its agent last reported) per registration.
 
     An agent that has never reported counts from when it was registered. A
-    registration whose host is gone is skipped.
+    registration whose host is gone is skipped. Nothing counts from before
+    `watching_since`, when Steward last started watching without a break: an
+    agent can't report while Steward is down, so that time is not its silence
+    (#5441). An agent that is really gone reads as offline once it has been
+    quiet that long while Steward watched. `arrived` (arrivals(app)) counts a
+    post Steward received but couldn't store as the agent seen (#5514).
+
+    `already_silent` names hosts whose last recorded figure was already past
+    the offline window: their agent was gone before Steward stopped, so their
+    whole silence counts. Counting them from `watching_since` read them as
+    just seen, which resolved their alert on every restart and fired it again
+    three minutes later (#5719).
+
+    `explained_until` ({host id: time}) is a stop the agent announced, a reboot
+    or its own restart, plus its grace (#5756): until then the agent reads as
+    just seen. A host still silent past it reads its whole silence, so agent
+    offline fires as the grace runs out, for silence nobody explained.
     """
     out = []
     for r in regs:
         name = host_names.get(r.host_id)
-        seen = r.last_seen_at or r.created_at
+        seen = max((t for t in (r.last_seen_at or r.created_at, (arrived or {}).get(r.host_id))
+                    if t is not None), default=None)
         if name is None or seen is None:
             continue
+        if watching_since is not None and name not in already_silent:
+            seen = max(seen, watching_since)
+        if r.host_id in (explained_until or {}):
+            seen = max(seen, explained_until[r.host_id])
         out.append((name, max(0.0, (now - seen).total_seconds())))
     return out
 
@@ -38,13 +82,25 @@ async def record_seconds_since_seen(app) -> None:
     async with app.db_sessionmaker() as session:
         async with session.begin():
             regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
+            watching_since = watch(app, "host_agent_seconds_since_seen").tick(now)
             if not regs:
                 return
             host_names = dict((await session.execute(
                 select(Host.id, Host.name).where(Host.id.in_([r.host_id for r in regs]))
             )).all())
+            # Agents whose last figure was already past the offline window keep
+            # counting through a restart of Steward (#5719).
+            stale_after = agent_stale_after_seconds(app.config)
+            already_silent = set((await session.execute(
+                select(PluginMetricLatest.resource_name).where(
+                    PluginMetricLatest.source_module == "host_agent",
+                    PluginMetricLatest.metric_name == "seconds_since_seen",
+                    PluginMetricLatest.resource_name.in_(list(host_names.values())),
+                    PluginMetricLatest.value > stale_after))).scalars())
+            explained = await announced_stops(session, now, restart_grace_seconds(app.config))
             batch = MetricBatch()
-            for name, seconds in seconds_since_seen(regs, host_names, now):
+            for name, seconds in seconds_since_seen(regs, host_names, now, watching_since,
+                                                    arrivals(app), already_silent, explained):
                 batch.add(source_module="host_agent", resource_name=name,
                           metric_name="seconds_since_seen", value=seconds, recorded_at=now)
             await batch.record(session)
@@ -63,3 +119,60 @@ def make_task(app) -> ScheduledTask:
         interval_seconds=60,
         run_on_startup=False,
     )
+
+
+def make_sensor_task(app) -> ScheduledTask:
+    """Re-judges which temperature sensors are thermometers (#5783), hourly
+    and at startup, from the hourly averages: verdicts live in memory, so a
+    restart rebuilds them before the first hour is out."""
+    async def _judge():
+        from . import sensors
+        try:
+            async with app.db_sessionmaker() as session:
+                found = await sensors.refresh(session, datetime.now(timezone.utc))
+            for host, labels in found.items():
+                logger.info("host_agent: %s sensors not counted as temperature: %s",
+                            host, ", ".join(sorted(labels)))
+        except Exception:
+            logger.exception("host_agent: judging temperature sensors failed")
+
+    return ScheduledTask(
+        name="host_agent_sensors",
+        coro_factory=_judge,
+        interval_seconds=3600,
+        run_on_startup=True,
+    )
+
+
+def make_maintenance_task(app) -> ScheduledTask:
+    """Gives up on maintenance jobs past their deadline (#5767), starts due
+    maintenance plans and moves their runs on one host at a time (#5768), and
+    records each host's maintenance_failed for an alert rule. A job's steps
+    move on from its agent's posts; the tick moves on only Steward's own waits
+    (the 5 minutes after a drain, the containers seen running again, #6098)."""
+    async def _tick():
+        from . import cadence, maintenance
+        now = datetime.now(timezone.utc)
+        try:
+            async with app.db_sessionmaker() as session:
+                async with session.begin():
+                    gave_up = await maintenance.tick(session, now)
+                    said = await cadence.tick(session, now)
+                    batch = MetricBatch()
+                    for name, value in (await cadence.failed_hosts(session, now)).items():
+                        batch.add(source_module="host_agent", resource_name=name,
+                                  metric_name="maintenance_failed", value=value, recorded_at=now)
+                    await batch.record(session)
+            if gave_up:
+                logger.warning("host_agent: %d maintenance job(s) timed out", gave_up)
+            for line in said:
+                logger.info("host_agent maintenance: %s", line)
+        except Exception:
+            logger.exception("host_agent: maintenance tick failed")
+
+    return ScheduledTask(
+        name="host_agent_maintenance",
+        coro_factory=_tick,
+        interval_seconds=60,
+        run_on_startup=False,
+    )

plugins/host_agent/senders.py

@@ -0,0 +1,113 @@
+"""Who sends a host's network traffic (#6115): its containers and NFS mounts,
+ranked against the host's own total over a window.
+
+A host's net_tx_bps (net_rx_bps) is what its physical interfaces carried; each
+container's comes from Docker ingest (resource "<host>/<service or container>") and each
+NFS mount's from the agent (resource "<host>:<mount>"). All three are read in
+the same time buckets, so the busiest bucket of the host's total can be put
+down to who was sending in it. What no container or mount accounts for is the
+host itself (an apt download, a backup run from the host, a swarm ingress
+hand-off) or a container's traffic that never left the host.
+
+Imports only the core metric model, like metrics_query, so the docker plugin
+being off leaves just the NFS mounts and the host.
+"""
+from __future__ import annotations
+
+import re
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import func, select
+
+from steward.models.metrics import PluginMetric, PluginMetricLatest
+
+SOURCE_MODULE = "host_agent"
+DIRECTIONS = ("tx", "rx")
+# Buckets a window is cut into: a 1 h window reads per 60 s sample, 24 h per
+# 12 minutes. Narrow the window to see a spike sharper.
+BUCKETS = 120
+# A swarm task's container, "<service>.<slot or node id>.<task id>", is shown
+# by its service, so its tasks over the window add up to one sender.
+_TASK = re.compile(r"^(?P<svc>.+?)\.(?:\d+|[a-z0-9]{25})\.[a-z0-9]{25}$")
+
+
+def sender_name(container: str) -> str:
+    m = _TASK.match(container)
+    return m["svc"] if m else container
+
+
+def rank(host: dict, senders: dict, limit: int = 10) -> dict:
+    """`host` is {bucket: bytes a second}; `senders` is {(kind, name):
+    {bucket: bytes a second}}. The host's average, its peak and when, and each
+    sender's average over the host's buckets (a bucket it is missing from
+    counts as 0), its own peak and what it sent in the host's peak bucket,
+    biggest at the host's peak first. `unaccounted_*` is what the host carried
+    beyond them all, never below 0."""
+    if not host:
+        return {"host": None, "senders": [], "more": 0, "unaccounted_avg_bps": None,
+                "unaccounted_at_peak_bps": None}
+    n = len(host)
+    peak_at = max(host, key=lambda b: host[b])
+    rows = []
+    for (kind, name), series in senders.items():
+        top = max(series, key=lambda b: series[b])
+        rows.append({"kind": kind, "name": name,
+                     "avg_bps": sum(v for b, v in series.items() if b in host) / n,
+                     "peak_bps": series[top], "peak_at": top,
+                     "at_host_peak_bps": series.get(peak_at, 0.0)})
+    rows.sort(key=lambda r: (r["at_host_peak_bps"], r["avg_bps"]), reverse=True)
+    host_avg = sum(host.values()) / n
+    return {
+        "host": {"avg_bps": host_avg, "peak_bps": host[peak_at], "peak_at": peak_at},
+        "senders": rows[:limit],
+        "more": max(0, len(rows) - limit),
+        "unaccounted_avg_bps": max(0.0, host_avg - sum(r["avg_bps"] for r in rows)),
+        "unaccounted_at_peak_bps": max(0.0, host[peak_at] - sum(r["at_host_peak_bps"] for r in rows)),
+    }
+
+
+async def senders(session, host_name: str, since: datetime, direction: str = "tx",
+                  limit: int = 10, raw_days: int = 7) -> dict:
+    """rank() over the raw samples since `since` (no earlier than the raw
+    window: the hourly rollup keeps no per-bucket sender to rank). Adds the
+    window actually read and its bucket width."""
+    if direction not in DIRECTIONS:
+        raise ValueError(f"direction is one of {', '.join(DIRECTIONS)}")
+    now = datetime.now(timezone.utc)
+    since = max(since, now - timedelta(days=raw_days))
+    width = max(60, int((now - since).total_seconds() / BUCKETS))
+    bucket = func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, width), PluginMetric.recorded_at,
+                           func.to_timestamp(0)).label("bucket")
+    metric = f"net_{direction}_bps"
+
+    async def named(source, metric_name, prefix):
+        # The resources under the host, from the small latest-value table: a
+        # prefix LIKE on plugin_metrics itself can't use its index (the
+        # column's collation isn't C) and reads the whole table.
+        return list((await session.execute(
+            select(PluginMetricLatest.resource_name).where(
+                PluginMetricLatest.source_module == source, PluginMetricLatest.metric_name == metric_name,
+                PluginMetricLatest.resource_name.startswith(prefix, autoescape=True),
+                PluginMetricLatest.recorded_at >= since))).scalars())
+
+    async def series(source, metric_name, resources):
+        if not resources:
+            return []
+        return (await session.execute(
+            select(PluginMetric.resource_name, bucket, func.avg(PluginMetric.value))
+            .where(PluginMetric.source_module == source, PluginMetric.resource_name.in_(resources),
+                   PluginMetric.metric_name == metric_name, PluginMetric.recorded_at >= since)
+            .group_by(PluginMetric.resource_name, bucket))).all()
+
+    host: dict = {}
+    for _, b, v in await series(SOURCE_MODULE, metric, [host_name]):
+        host[b] = float(v)
+    out: dict = {}
+    for res, b, v in await series("docker", metric, await named("docker", metric, host_name + "/")):
+        key = ("container", sender_name(res[len(host_name) + 1:]))
+        out.setdefault(key, {})
+        out[key][b] = out[key].get(b, 0.0) + float(v)          # a service's tasks add up
+    nfs = f"nfs_{direction}_bps"
+    for res, b, v in await series(SOURCE_MODULE, nfs, await named(SOURCE_MODULE, nfs, host_name + ":/")):
+        out.setdefault(("nfs", res[len(host_name) + 1:]), {})[b] = float(v)
+    return {"since": since, "bucket_s": width, "direction": direction, **rank(host, out, limit)}