feat(docker): the Swarm page is laid out by node: role, what its containers use, and the services placed on each (#5303)

#5303 "Swarm page laid out by node: role, resources and the tasks placed on each"

A verdict line first, then the nodes that are down or drained and the
services short of replicas, then one card per node. A node with a Steward
agent shows what its swarm tasks use against the machine's cores and memory,
service by service. A node without one lists Swarm's placement, by count.
The services table moved to the Docker overview, where it can be filtered.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.forgejo/workflows/ci.yml

@@ -14,6 +14,17 @@ on:
   # pull_request intentionally absent — push on [dev, main] already fires CI for
   # every dev commit and dev→main PRs. Single-operator repo, no fork PRs.
 
+# Serialize CI runs per ref so the publish lane never shares the runner's docker
+# daemon with another run. Two publishes racing on one daemon evict each other's
+# freshly-built image mid-push, breaking the post-build tag/push (issue #1093).
+# cancel-in-progress:false is deliberate — rule 46 requires EVERY dev/main push to
+# publish its own immutable :<sha> image, so a superseded run must still run to
+# completion (never cancelled) to emit its SHA. (Forgejo honors `concurrency:` —
+# CI-runner's build workflows use it.)
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: false
+
 jobs:
   # Fast-fail lint lane. ruff is pre-installed in the ci-python image, so this
   # runs with NO dependency install and surfaces lint bounces in seconds.
@@ -69,14 +80,24 @@ jobs:
         # act_runner (swarm-runner) puts service containers on the default bridge
         # with no embedded DNS, so the hostname `postgres` won't resolve — we
         # discover the service container's bridge IP via the docker socket. The
-        # job name `integration` (no underscore) is the docker-ps name filter.
+        # job container and its Postgres share a task-number name prefix.
         run: |
           set -euxo pipefail
           echo "=== container landscape (diagnostic for filter scoping) ==="
           docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
           echo "=== end landscape ==="
-          PG=$(docker ps --filter "name=integration" --filter "ancestor=postgres:16-alpine" -q | head -n1)
-          test -n "$PG"
+          # The runner's daemon is shared: another repo's integration job can be
+          # running beside this one, and `name=integration` matches its Postgres
+          # too (run 8358 took the wrong one and every test failed to connect).
+          # Scope to this job's own task: act_runner names every container of a
+          # task GITEA-ACTIONS-TASK-<n>-…, so read <n> from our own container
+          # (its id is in the /etc/hostname bind mount) and require one match.
+          SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
+          SELF=${SELF:-$(hostname)}
+          TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
+          test -n "$TASK"
+          PG=$(docker ps --filter "name=$TASK" --filter "ancestor=postgres:16-alpine" -q)
+          test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
           PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
           test -n "$PG_IP"
           export STEWARD_DATABASE_URL="postgresql+asyncpg://steward:steward@$PG_IP:5432/steward"
@@ -111,19 +132,118 @@ jobs:
         # 401s with it. REGISTRY_TOKEN is a repo Actions secret holding a
         # Forgejo token scoped read:package + write:package.
         run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.fabledsword.com -u bvandeusen --password-stdin
-      - name: Build + push (commit-SHA always; :dev on dev, :latest on main)
+      - name: Build (commit-SHA always; :dev on dev, :latest on main)
         run: |
-          set -euxo pipefail
+          set -eux
+          IMAGE=git.fabledsword.com/bvandeusen/steward
+          # The moving tag for this ref: dev→:dev, main→:latest (rule 46). main IS
+          # the production line, so :latest tracks main's tip; there is no :main.
+          MOVING=""
+          if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
+            MOVING="dev"
+          elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
+            MOVING="latest"
+          fi
+          # Tag BOTH targets at build time (mirrors CI-runner's build workflows):
+          # one `docker build -t :<sha> -t :<moving>` points both tags at the image
+          # atomically, so we never run a post-push `docker tag` of an image a
+          # concurrent run could have evicted from the shared daemon (issue #1093).
+          if [ -n "$MOVING" ]; then
+            docker build -t "$IMAGE:${{ github.sha }}" -t "$IMAGE:$MOVING" .
+          else
+            docker build -t "$IMAGE:${{ github.sha }}" .
+          fi
+
+      # Rule 164's check, BETWEEN build and push (rule 177): an image that
+      # can't boot, migrate and serve with no internet never reaches a tag.
+      # An --internal network lets it reach its own Postgres and nothing else;
+      # --network none can't work for an app that needs a database to start.
+      # Embedded DNS works on a user-defined network, unlike the services:
+      # containers above (rule 79), so the app finds Postgres by name.
+      - name: The image boots with no internet
+        env:
+          SMOKE: steward-smoke-${{ github.run_id }}
+        run: |
+          set -eux
+          IMAGE=git.fabledsword.com/bvandeusen/steward:${{ github.sha }}
+          docker network create --internal "$SMOKE-net"
+          docker run -d --name "$SMOKE-pg" --network "$SMOKE-net" \
+            -e POSTGRES_USER=steward -e POSTGRES_PASSWORD=steward -e POSTGRES_DB=steward \
+            postgres:16-alpine
+          # Waits with deadlines (rule 156); pg_isready, not /dev/tcp, which is
+          # a bash builtin and never succeeds under sh.
+          ready=""
+          for _ in $(seq 1 60); do
+            if docker exec "$SMOKE-pg" pg_isready -U steward -d steward -q; then ready=1; break; fi
+            sleep 2
+          done
+          test -n "$ready" || { echo "FATAL: postgres not ready within 120s"; exit 1; }
+
+          docker run -d --name "$SMOKE-app" --network "$SMOKE-net" \
+            -e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/steward" \
+            -e STEWARD_SECRET_KEY=smoke-test-placeholder \
+            "$IMAGE"
+          # The image has no curl; its own Python fetches. Prints the body.
+          fetch() {
+            docker exec "$SMOKE-app" python -c 'import sys, urllib.request
+          r = urllib.request.urlopen("http://127.0.0.1:5000" + sys.argv[1], timeout=5)
+          sys.stdout.write(r.read().decode())' "$1"
+          }
+          # Migrations run before the server starts, so /health answering
+          # means the schema applied against a real database.
+          up=""
+          for _ in $(seq 1 60); do
+            if fetch /health >/dev/null 2>&1; then up=1; break; fi
+            sleep 2
+          done
+          if [ -z "$up" ]; then
+            echo "FATAL: Steward never answered /health with no internet access"
+            docker logs "$SMOKE-app" 2>&1 | tail -80
+            exit 1
+          fi
+          fetch /health | grep -q '"ok"'
+
+          # tini is PID 1, so zombies are reaped and SIGTERM reaches Steward.
+          test "$(docker exec "$SMOKE-app" cat /proc/1/comm)" = "tini"
+
+          # A fresh database lands on first-run setup: a real page, its assets
+          # served by Steward itself, and no third-party host in it (rule 155).
+          fetch / > /tmp/smoke-page.html
+          grep -qi "<!doctype html" /tmp/smoke-page.html
+          grep -q "/static/" /tmp/smoke-page.html
+          if grep -Eqi 'https?://(fonts\.|cdn\.|unpkg\.|cdnjs\.|ajax\.)' /tmp/smoke-page.html; then
+            echo "FATAL: the page references a third-party host"
+            grep -Eoi 'https?://[^"'"'"' ]+' /tmp/smoke-page.html
+            exit 1
+          fi
+
+          # A stop is a clean shutdown: SIGTERM through tini to Hypercorn, which
+          # finishes within its graceful timeout and exits 0.
+          docker stop -t 30 "$SMOKE-app"
+          code="$(docker inspect -f '{{.State.ExitCode}}' "$SMOKE-app")"
+          test "$code" = "0" || { echo "FATAL: Steward exited $code on SIGTERM"; exit 1; }
+
+      - name: Push
+        run: |
+          set -eux
           IMAGE=git.fabledsword.com/bvandeusen/steward
-          docker build -t "$IMAGE:${{ github.sha }}" .
           docker push "$IMAGE:${{ github.sha }}"
           if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
-            docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:dev"
             docker push "$IMAGE:dev"
           elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
-            docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:latest"
             docker push "$IMAGE:latest"
           fi
+
+      - name: Smoke logs
+        if: always()
+        run: docker logs "steward-smoke-${{ github.run_id }}-app" 2>&1 | tail -60 || true
+
+      # A failed smoke must not leave a Postgres and an app on the shared daemon.
+      - name: Smoke clean-up
+        if: always()
+        run: |
+          docker rm -f "steward-smoke-${{ github.run_id }}-app" "steward-smoke-${{ github.run_id }}-pg" >/dev/null 2>&1 || true
+          docker network rm "steward-smoke-${{ github.run_id }}-net" >/dev/null 2>&1 || true
       - name: Prune dangling layers
         if: always()
         run: docker image prune -f

Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.13-slim
+# The Python CI tests on (ci-python:3.14), so the image runs what was tested.
+FROM python:3.14-slim
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
@@ -10,6 +11,11 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \
         # sshpass — Ansible shells out to it for first-contact password SSH
         # (provisioning a fresh host before the managed key is installed)
         sshpass \
+        # git — clones and pulls Ansible git sources (Settings > Ansible)
+        git \
+        # tini — PID 1: reaps the zombies ssh and ansible leave behind and
+        # forwards SIGTERM to Steward so a stop is a graceful shutdown
+        tini \
     && rm -rf /var/lib/apt/lists/*
 
 # Upgrade pip to suppress the version notice
@@ -19,10 +25,11 @@ WORKDIR /app
 
 COPY pyproject.toml .
 COPY steward/ steward/
-# Bundle the extras whose first-party plugins ship in the image: [ansible] for
-# the playbook runner, [snmp] (pysnmp) for the SNMP poller. Without them those
-# bundled plugins load but silently no-op for want of their runtime dependency.
-RUN pip install --no-cache-dir '.[ansible,snmp]'
+# Bundle the extras whose features ship in the image: [ansible] for the
+# playbook runner, [snmp] (pysnmp) for the SNMP poller, [ldap] (ldap3, pure
+# Python) for LDAP sign-in under Settings -> Auth. Without them those features
+# load but fail for want of their runtime dependency.
+RUN pip install --no-cache-dir '.[ansible,snmp,ldap]'
 
 COPY alembic.ini .
 # First-party plugins ship inside the image (bundled root at /app/plugins).
@@ -36,10 +43,10 @@ RUN chmod +x /entrypoint.sh
 # so the entrypoint can fix up /data permissions if needed, then drops to
 # 'app' (uid 1000) via gosu before launching steward.
 RUN useradd -m -u 1000 app \
-    && mkdir -p /data/playbook_cache /data/plugins \
+    && mkdir -p /data/ansible/sources /data/ansible/runs /data/ansible/playbooks /data/plugins \
     && chown -R app:app /data /app
 
 EXPOSE 5000
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/usr/bin/tini", "--", "/entrypoint.sh"]
 CMD ["steward", "--host", "0.0.0.0", "--port", "5000"]

ci-requirements.md

@@ -46,7 +46,7 @@ git.fabledsword.com/bvandeusen/ci-python:3.14
 
 - Steward has **no frontend and no Redis/Celery** (no external workers), so there
   is no frontend-build lane and the only service container is Postgres.
-- Integration uses Forgejo Actions `services:` + a socket-discovered bridge IP
+- Integration uses Fabled-Git Actions `services:` + a socket-discovered bridge IP
   because `act_runner` (swarm-runner v0.6+) puts services on the default bridge
   with no embedded DNS. FabledCurator's `ci.yml` is the canonical example of the
   pattern; Steward's is the same shape minus Redis and minus sharding.

config.example.yaml

@@ -13,6 +13,14 @@
 database:
   url: "postgresql+asyncpg://steward:password@localhost/steward"
 
+  # Optional: seconds to wait at startup for the database to become reachable
+  # before giving up (default: 60). Steward retries with backoff rather than
+  # crashing when the DB is merely slow to arrive — replaying WAL after an
+  # unclean shutdown, or container DNS not yet resolving after a host reboot.
+  # Raise it if your database is routinely slower than this to accept
+  # connections. Env var: STEWARD_DB_CONNECT_TIMEOUT
+  # connect_timeout: 60
+
 # Optional: override the auto-generated secret key.
 # If not set, a key is auto-generated on first run and saved to /data/secret.key.
 # secret_key: "change-me-to-a-random-string"

docs/architecture.md

@@ -44,6 +44,30 @@ All routes are Quart Blueprints registered in `app.py`:
 
 Plugin blueprints are mounted automatically by `load_plugins()` using the plugin directory name as the URL prefix.
 
+`/ai-access/` (`ai_access_bp`, `steward/ai_access/routes.py`) is where a signed-in user manages their API tokens.
+
+### `/mcp`: outside the blueprint pipeline
+
+`mount_mcp(app)` (`steward/mcp/server.py`) wraps `app.asgi_app`. A request to
+`/mcp` is handed to the MCP SDK's streamable-HTTP app before Quart sees it, so
+no blueprint, session cookie or `before_request` hook runs for it. Each
+request then goes through these steps:
+
+1. **Database gate.** If the database is unreachable, the request gets a 503
+   with a readable message.
+2. **Bearer resolution.** `api_tokens.resolve_token()` turns the token into a
+   `Caller` (user, effective role, token id). A missing, revoked or expired
+   token gets a 401.
+3. **Caller context.** The `Caller` and the app are set in ContextVars
+   (`steward/mcp/_context.py`). Tools read them through `current_caller()` and
+   `mcp_app()`, because Quart's `current_app` and `session` do not exist there.
+4. **Role gate.** `StewardMCPServer.refusal()` hides and refuses tools above
+   the caller's role, and plugin tools whose plugin is disabled.
+
+The SDK's session manager runs in Quart's `before_serving`/`after_serving`
+lifecycle. The server is stateless: no MCP session outlives a request. See
+[MCP server](core/mcp.md) for the security model and the tools.
+
 ---
 
 ## Scheduler

docs/core/alerting.md

@@ -8,7 +8,7 @@ Alert rules evaluate every metric that flows through `record_metric()`. There is
 
 When any monitor or plugin calls `record_metric()`:
 
-1. The metric value is written to the `plugin_metrics` table
+1. The metric value is written to the `plugin_metrics` table, and `plugin_metric_latest` moves to it if it is the newest sample of that metric
 2. All enabled `AlertRule` rows matching `(source_module, resource_name, metric_name)` are loaded
 3. Each matching rule is evaluated against the new value
 4. If a state transition occurs, an `AlertEvent` row is written
@@ -136,3 +136,4 @@ Defined in `steward/models/alerts.py`:
 - **`alert_states`** — one row per rule, tracks current state and consecutive failure count
 - **`alert_events`** — append-only log of all state transitions and notification outcomes
 - **`plugin_metrics`** — all metric values written by any monitor or plugin
+- **`plugin_metric_latest`** — the newest sample of each (source, resource, metric). Pages that show "what it is now" read this instead of searching history; retention drops a row once its sample leaves the raw window

docs/core/ansible.md

@@ -31,10 +31,9 @@ ansible.sources:
     url: "https://github.com/example/infra.git"
     branch: "main"
     pull_interval_seconds: 300
-    cache_path: "/data/playbook_cache/infra-repo"
 ```
 
-Git sources register a `ScheduledTask` named `ansible_git_pull_<name>` that runs `git pull` on the configured interval.
+Each git source is checked out under `/data/ansible/sources/<name>` (override with `STEWARD_ANSIBLE_SOURCE_DIR`), on the persistent volume, so a redeploy keeps it. One scheduled task, `ansible_git_pulls`, ticks every 30 s and pulls each source once its `pull_interval_seconds` has passed; a source added in Settings is pulled within a tick, with no restart. A clone or pull that runs past 120 s is cut (its process group terminated), a clone that fails is removed so the next attempt starts clean, and git never prompts for credentials.
 
 ---
 
@@ -102,12 +101,20 @@ If a client connects after the run has already completed, the endpoint immediate
 
 | Status | Description |
 |---|---|
+| `queued` | Waiting for a host another run holds, or for a run slot |
 | `running` | Execution in progress |
 | `success` | Playbook exited 0 |
 | `failed` | Playbook exited non-zero |
 | `interrupted` | App restarted while run was in progress |
+| `cancelled` | Stopped by an operator, or by Steward shutting down mid-run |
 
-On startup, the app marks any runs still in `running` state as `interrupted` (see `_mark_interrupted_runs()` in `app.py`).
+On startup, the app marks any runs still `running` or `queued` as `interrupted` (see `_mark_interrupted_runs()` in `app.py`).
+
+### Overlapping runs
+
+- **One run per host.** Before it takes a run slot, a run claims every host it touches. Hosts come from the inventory Steward writes, keyed by address. A run on a source's own inventory file claims that file, because Steward doesn't read its hosts. While another run holds one of its hosts, the run shows `queued` and says which run it waits for. After an hour of waiting it fails and says so.
+- **One run at a time per schedule.** A schedule that falls due while its previous run is still `queued` or `running` skips that firing. The Schedules page shows "Skipped: the previous run was still going" beside that run. The next firing comes an interval later.
+- **One action run at a time per alert rule.** When a rule fires again while the action run from an earlier firing is still going, no new run starts. The firing's event says it was skipped and names the run.
 
 Output stored in the DB is capped at 1 MB. If truncated, `[output truncated]` is appended to the DB column, but the live SSE stream continues unaffected.
 

docs/core/mcp.md

@@ -0,0 +1,132 @@
+# MCP server
+
+Steward serves the [Model Context Protocol](https://modelcontextprotocol.io) at
+`/mcp`. An AI client connected to it can look into everything Steward watches
+(hosts, monitors, alerts, metrics, containers, playbook runs) and, with a
+high enough role, change it. A token's role sets that ceiling.
+
+**Source:** `steward/mcp/`. The tools live in `steward/mcp/tools/`, and a
+plugin's tools live in `plugins/<name>/mcp_tools.py`.
+**Transport:** streamable HTTP, stateless. Every request carries its own
+token, so a client survives a Steward restart without reconnecting.
+
+---
+
+## Connecting
+
+1. In Steward, open **AI access** and create a token. Pick the lowest role
+   that does the job.
+2. Copy the token (it is shown once) and use one of the recipes on that
+   page. They carry the real endpoint URL and the new token. For Claude Code:
+
+   ```sh
+   claude mcp add --transport http steward https://steward.example/mcp \
+     --header "Authorization: Bearer stw_…"
+   ```
+
+3. Ask the client to call `whoami`. It answers with the user, the effective
+   role and what that role allows.
+
+Claude Desktop connects through `mcp-remote`. Open WebUI and other clients
+need an MCP server with the **Streamable HTTP** transport and bearer
+authentication. The AI access page shows the recipes for both.
+
+### Behind a reverse proxy
+
+- The URL on the AI access page is **Settings → General → Public base URL**
+  plus `/mcp`. If that setting is empty, the URL is built from the request's
+  host instead.
+- The proxy must pass the `Authorization` header through unchanged.
+- It must not buffer responses. Replies may arrive as server-sent events
+  (`text/event-stream`). For nginx, set `proxy_buffering off;` on the
+  location.
+- Steward does no Host-header (DNS-rebinding) check on `/mcp`, because the
+  bearer token is the security boundary. Serve it over HTTPS.
+
+---
+
+## Security model
+
+| Rule | How it is enforced |
+|---|---|
+| Bearer tokens only, no OAuth | `steward/core/api_tokens.py`. Only a hash is stored, and the token is shown once. |
+| A token never acts above its user | Effective role = min(token role, user's **current** role). Lowering a user lowers every token they hold; disabling the user stops them all. |
+| Default deny | A tool is reachable only if it was registered with a role (`register_tool(fn, role=..., read_only=...)`). `tools/list` shows a caller only the tools their role allows. A call above the role is refused, and the refusal names the role it needs. |
+| Plugin tools follow the plugin | A disabled plugin's tools are hidden and refused. |
+| Dry run first | `run_playbook`, `create_schedule` and `set_alert_action` default to Ansible check mode (`--check --diff`). Changes are applied only when the caller passes `check=false`. A playbook its author flagged `# steward:confirm: true` also needs `confirmed=true`. `prune_docker` previews unless `check=false`. |
+| Secrets stay put | Playbook variables Steward detects as secret reach the run but are never stored, audited or returned. Schedules and alert actions refuse them, since they would have to store them. Inventory variables are listed by name only, and HTTP monitor header values are redacted. |
+| Every change is audited | Each write records an audit event as `<user> via MCP`, with the token id in its detail. `list_audit_events(user="<user>")` includes those events. |
+| Unknown arguments are refused | A misspelt argument is rejected with a "did you mean", rather than silently dropped. |
+
+Tokens never mint or revoke tokens; that happens only on the AI access
+page, in a browser session.
+
+---
+
+## Tools by role
+
+Each role includes the tools of the roles above it in this list.
+
+### Viewer: read everything
+
+| Area | Tools |
+|---|---|
+| Start here | `whoami`, `get_overview` (what is wrong now, ranked, each finding with the call that shows its evidence) |
+| Hosts and metrics | `list_hosts`, `get_host`, `get_host_metrics`, `query_metrics` |
+| Monitors | `list_monitors`, `get_monitor` |
+| Alerts | `list_alerts`, `get_alert_history`, `list_alert_rules`, `get_alert_rule_options`, `list_maintenance_windows` |
+| Automation | `list_playbooks`, `get_playbook`, `list_inventory`, `list_ansible_runs`, `get_ansible_run`, `wait_for_run`, `list_schedules` |
+| Audit | `list_audit_events` |
+| Docker plugin | `list_containers`, `get_container`, `list_swarm_services`, `search_container_logs`, `get_docker_disk_usage` |
+| Host agent plugin | `list_agents` |
+
+### Operator: run playbooks, change monitoring
+
+| Area | Tools |
+|---|---|
+| Automation | `run_playbook`, `cancel_ansible_run`, `create_schedule`, `update_schedule`, `delete_schedule` |
+| Monitors | `create_monitor`, `update_monitor`, `set_monitor_enabled` |
+| Alerts | `create_alert_rule`, `update_alert_rule`, `acknowledge_alert`, `create_maintenance_window`, `end_maintenance_window` |
+
+### Admin: destructive and remediation
+
+`set_alert_action` (the playbook a rule runs when it fires), `delete_monitor`,
+`delete_alert_rule`, `prune_docker`.
+
+The client sees each tool's full contract: arguments, what an empty value
+means, and what comes back. That contract is the tool's docstring, so read the
+docstrings in `steward/mcp/tools/` for the detail.
+
+---
+
+## Response conventions
+
+Every tool follows these conventions (`steward/mcp/tools/_shape.py`):
+
+- **Times** are ISO-8601 UTC with a `Z`. Anything with a "when" also carries
+  `age_s`, so the client never does clock arithmetic.
+- **Lists are bounded.** A list response carries `total` and `truncated`. When
+  it was cut, `more` names the argument that fetches the rest. Nothing is cut
+  silently.
+- **Time series** come back as a summary (min/avg/max/p95/last) plus a series
+  of at most 60 points, never raw rows.
+- **Lookup by name or id.** A miss suggests close matches and names the tool
+  that lists the valid choices.
+- **Refusals are readable.** Bad input becomes a tool error that says what was
+  wrong, why, and what to do instead, and nothing is changed. The tools share
+  their validation with the web forms (`monitors/service.py`,
+  `alerts/service.py`), so a value the UI refuses is refused here for the same
+  reason.
+- **Database outages** return a plain "Steward cannot reach its database"
+  rather than a stack trace.
+
+---
+
+## Adding tools
+
+Core tools: add an `async def` to a module in `steward/mcp/tools/` and list it
+in that module's `register()` with its minimum role and `read_only`. A plugin
+exposes `register_mcp_tools(register)`, and optionally `mcp_overview(db)` to
+contribute findings to `get_overview`. See
+[Writing a plugin](../plugins/writing-a-plugin.md). A tool that changes
+anything calls `audit_mcp(...)` from `steward/mcp/tools/_audit.py`.

docs/plugins/host-agent-design.md

@@ -74,8 +74,15 @@ token = a1b2c3d4...
 interval_seconds = 30
 hostname = myhost              # optional; defaults to uname -n
 mounts = /, /mnt/data          # optional; defaults to all real mounts (excluding tmpfs/devtmpfs/etc.)
+docker_logs_enabled = true     # optional; on by default — per-host container-log kill-switch
+docker_log_exclude = watchtower, noisy-svc   # optional; container names whose logs this host skips collecting
 ```
 
+Container-log collection is on by default (no config needed). The two keys above
+are the per-host opt-outs; the operator-facing global toggle + exclude live in
+Settings and are enforced server-side (the push model has no channel to command
+an agent), so they don't require touching a host's conf.
+
 ### Agent internals (function list, not classes)
 
 - `read_config(path)` — parses the conf file into a dict.
@@ -85,10 +92,17 @@ mounts = /, /mnt/data          # optional; defaults to all real mounts (excludin
 - `collect_load()` — reads `/proc/loadavg`, returns `[1m, 5m, 15m]`.
 - `collect_uptime()` — reads `/proc/uptime`, returns seconds since boot (int).
 - `collect_metadata()` — `os.uname()` for kernel + arch, `/etc/os-release` for distro. Called once at startup and cached.
+- `collect_docker_logs(socket, containers, state, exclude)` — (m79) per running
+  container, fetches new log lines over the Docker socket with an incremental
+  `since` cursor kept per container in `state` (a container's first interval
+  seeds from a short tail). Demuxes the multiplexed stream, parses each line's
+  RFC3339 timestamp, caps the whole batch at a byte limit (a marker line records
+  a truncation; the deferred lines come next interval). Folded into the sample as
+  `docker_logs`; omitted when empty.
 - `build_payload()` — assembles a snapshot from all collectors into one dict.
 - `post_payload(url, token, payloads)` — POSTs a list of samples, returns success/failure.
 - `RingBuffer(maxlen=20)` — tiny FIFO wrapper, drops oldest when full.
-- `main_loop()` — the 30s loop: collect → try POST → on failure push to buffer + backoff → on success flush buffer.
+- `main_loop()` — the 30s loop: collect → try POST → on failure push to buffer + backoff → on success flush buffer. Container logs are stripped from a sample before it's buffered (metrics survive an outage; stale logs are dropped).
 
 **Target: ~300 lines total including docstrings.** More than that is a smell that the agent is over-scoping.
 
@@ -143,7 +157,6 @@ plugins/host_agent/
 │       └── host_agent_001_initial.py    # creates host_agent_registrations table
 ├── templates/
 │   ├── detail.html                      # per-host detail page
-│   ├── widget_table.html                # dashboard fleet table widget
 │   ├── widget_history.html              # dashboard history chart widget
 │   ├── settings_list.html               # plugin settings: list of registered hosts
 │   └── install.sh.j2                    # install script template (Jinja)
@@ -255,6 +268,7 @@ Content-Type: application/json
 - **`metadata` is sent on every POST**, not just on change. Server-side diff detects actual changes and only writes on change. Cost per POST is one dict — negligible. Benefit: server can cleanly detect agent restarts.
 - **Raw bytes, not percentages, for memory and storage.** Percentages are derived server-side. Changing the "what counts as used" math doesn't require re-releasing the agent.
 - **CPU is the one exception** — reported as a percentage because it's inherently a derivative (delta over time), not a snapshot. The agent must sample twice to compute it.
+- **Container logs (m79) ride in the same push** as `docker_logs`: a list of `{container, stream, ts, line}` records — incremental since the previous interval. This pushes the *same direction* as metrics, so batched log history needs no inbound channel (only sub-second live-follow would). The server ingests them into `docker_logs`, enforces the global toggle + exclude list on ingest, and bounds storage with a per-container size+age ring. `docker_logs` is omitted when there's nothing new.
 
 ### Server expansion into `PluginMetric` rows
 
@@ -423,7 +437,7 @@ echo "Logs:          journalctl -u steward-agent -f"
 
 ### Dashboard widgets (`steward/core/widgets.py`)
 
-- **`host_resources`** — table widget. One row per monitored host: name, CPU %, mem %, disk % (worst mount), load 1m, "last seen Xs ago" with red/yellow/green coloring. Fleet glance.
+- The fleet glance (CPU, mem, disk /, load 1m, stale mark) is the core **Hosts — Overview** widget (`hosts_overview`); the old `host_resources` table was folded into it (#5213).
 - **`host_resource_history`** — chart widget for one host. CPU / mem / disk over a selectable time range (1h, 6h, 24h, 7d). Same pattern as every other history widget — Chart.js.
 
 ### Per-host detail page

docs/plugins/writing-a-plugin.md

@@ -323,6 +323,61 @@ Role hierarchy: `admin > operator > viewer`. Requiring `viewer` grants access to
 
 ---
 
+## Agent Tools (MCP)
+
+Steward serves an MCP endpoint at `/mcp` that AI clients reach with a personal API token created on the AI access page. A plugin adds tools to it by exporting an optional `register_mcp_tools(register)` from `__init__.py`:
+
+```python
+# plugins/myplugin/__init__.py
+def register_mcp_tools(register) -> None:
+    from .mcp_tools import list_targets, rescan_target
+    from steward.models.users import UserRole
+    register(list_targets, role=UserRole.viewer, read_only=True)
+    register(rescan_target, role=UserRole.operator, read_only=False)
+```
+
+```python
+# plugins/myplugin/mcp_tools.py
+from steward.mcp._context import current_caller, mcp_app
+
+
+async def list_targets(limit: int = 50) -> dict:
+    """What this tool is for, when to use it over its siblings, what each
+    argument's empty value means, and what comes back. This docstring is the
+    tool's description: it is all the agent knows about it."""
+    app = mcp_app()                 # Quart's current_app does not exist here
+    async with app.db_sessionmaker() as db:
+        ...
+    return {"targets": [...], "total": n, "truncated": n > limit}
+```
+
+- `role` and `read_only` are required. A tool is listed and callable only for callers whose role reaches `role`, and only while the plugin is enabled.
+- Tools run outside Quart's request pipeline: use `mcp_app()` and `current_caller()` from `steward.mcp._context`, never `current_app` or `session`.
+- Refuse by raising `ValueError("what was wrong, and what to do instead")`. The agent reads that text. Any other exception reaches it only as a generic failure.
+- Tool names are global across core and every plugin; a name already taken fails registration and the plugin contributes no tools (logged).
+- Hot-reloading the plugin replaces its tools.
+- Follow the response conventions the core tools use (`steward/mcp/tools/_shape.py`): times via `iso()` plus `age_s()`, lists through `bounded()` so a cut is never silent, series through `summarize()`, and lookups by name or id through `pick()`.
+
+### Contributing to get_overview
+
+`get_overview` is the agent's first call: what is wrong right now, ranked. A plugin adds its own problems to it by exporting an optional `mcp_overview(db)` that returns findings built with `steward.mcp.tools.overview.finding`:
+
+```python
+# plugins/myplugin/__init__.py
+async def mcp_overview(db) -> list[dict]:
+    from steward.mcp.tools.overview import finding
+    return [
+        finding("warning", "target_unreachable", t.name, f"{t.name} has not answered for {t.misses} scans",
+                since=t.last_ok_at, ids={"target_id": t.id}, next=f'get_target(target="{t.id}")')
+        for t in await _unreachable_targets(db)
+    ]
+```
+
+- `severity` is `"critical"` (something is down or about to be) or `"warning"` (degraded, or heading that way). Report problems only; a healthy plugin returns `[]`.
+- It is called only while the plugin is enabled, with its own session. If it raises, the overview still returns, and names the plugin under `unavailable`.
+
+---
+
 ## Publishing to the Catalog
 
 The official plugin catalog is hosted at `https://git.fabledsword.com/bvandeusen/Steward-plugins`. Anyone can submit a plugin by opening a pull request — first-party and third-party plugins are treated identically by the catalog system.
@@ -413,6 +468,7 @@ Hot-reload works reliably for brand-new plugin installs. Updates to already-acti
 - [ ] Revision IDs prefixed with plugin name
 - [ ] `record_metric()` called inside `session.begin()`
 - [ ] Routes use `@require_role` decorator
+- [ ] MCP tools (if any) registered via `register_mcp_tools(register)` with a role and `read_only`
 - [ ] Templates namespaced under `templates/myplugin/`
 - [ ] Plugin enabled in app settings
 - [ ] `index.yaml` entry added with `download_url` and `checksum_sha256`

docs/reference/code-map.md

@@ -89,6 +89,7 @@ Quick reference for where key functions, models, and entry points live in the co
 | `AlertState` | `steward/models/alerts.py` | `alert_states` |
 | `AlertEvent` | `steward/models/alerts.py` | `alert_events` |
 | `PluginMetric` | `steward/models/metrics.py` | `plugin_metrics` |
+| `PluginMetricLatest` | `steward/models/metrics.py` | `plugin_metric_latest` |
 | `AnsibleRun` | `steward/models/ansible.py` | `ansible_runs` |
 | `User` | `steward/models/users.py` | `users` |
 | `AppSetting` | `steward/models/settings.py` | `app_settings` |
@@ -136,3 +137,25 @@ Quick reference for where key functions, models, and entry points live in the co
 | `steward/migrations/versions/` | Core schema (hosts, users, monitors, alerts, app_settings) |
 | `plugins/traefik/migrations/versions/` | `traefik_metrics` table |
 | `alembic.ini` | Alembic config; `version_locations` lists all migration directories |
+
+---
+
+## MCP server and API tokens
+
+| What | File | Function / Class |
+|---|---|---|
+| Mount `/mcp` (ASGI dispatch, bearer auth, DB gate) | `steward/mcp/server.py` | `mount_mcp(app)` |
+| Server, role registry, refusals | `steward/mcp/server.py` | `StewardMCPServer`, `register_tool()`, `refusal()` |
+| Plugin tools and overview findings | `steward/mcp/server.py` | `register_plugin(name, module)` |
+| Caller and app inside a tool | `steward/mcp/_context.py` | `current_caller()`, `mcp_app()` |
+| Core tool modules | `steward/mcp/tools/` | `register_all()`; one module per area |
+| Response conventions | `steward/mcp/tools/_shape.py` | `iso`, `age_s`, `bounded`, `lookback`, `summarize`, `pick` |
+| Audit a write as "<user> via MCP" | `steward/mcp/tools/_audit.py` | `audit_mcp()` |
+| "What is wrong now" | `steward/mcp/tools/overview.py` | `get_overview()` |
+| Playbooks, runs, schedules, alert actions | `steward/mcp/tools/automation.py` | `run_playbook()`, `wait_for_run()`, … |
+| Monitor validation (forms and tools) | `steward/monitors/service.py` | `monitor_fields()`, `build_config()` |
+| Alert rule / window validation | `steward/alerts/service.py` | `rule_fields()`, `window_fields()`, `acknowledge()` |
+| Docker and host-agent tools | `plugins/docker/mcp_tools.py`, `plugins/host_agent/mcp_tools.py` | `register_tools()`, `mcp_overview()` |
+| Token model | `steward/models/api_tokens.py` | `ApiToken` |
+| Mint, resolve, revoke tokens | `steward/core/api_tokens.py` | `create_token()`, `resolve_token()`, `revoke_token()` |
+| Token management page and connect card | `steward/ai_access/routes.py`, `steward/templates/ai_access/` | `ai_access_bp` |

plugins/docker/__init__.py

@@ -49,6 +49,23 @@ def get_blueprint():
     return docker_bp
 
 
+def register_mcp_tools(register) -> None:
+    from .mcp_tools import register_tools
+    register_tools(register)
+
+
+async def mcp_overview(db) -> list[dict]:
+    from .mcp_tools import mcp_overview as findings
+    return await findings(db)
+
+
 def get_nav() -> list[dict]:
     # Surfaces in the sidebar's Infrastructure group (fleet-wide container view).
     return [{"label": "Docker", "href": "/plugins/docker/"}]
+
+
+def get_alert_metrics() -> dict[str, list[str]]:
+    """What ingest.py records per container (resource "<host>/<container>"):
+    restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 /
+    0.0 from the container's HEALTHCHECK, only for containers that define one."""
+    return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy"]}

plugins/docker/ingest.py

@@ -34,12 +34,16 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
     """Diff a fresh snapshot against stored per-container state → lifecycle events.
 
     Pure function (no DB) so it's unit-testable. `old_state` maps name → the
-    previously stored {status, health, oom_killed, exit_code}; `new_containers`
+    previously stored {status, health, oom_killed, exit_code, restart_count}; `new_containers`
     is the newest snapshot's list of container dicts. Returns a list of
     (container_name, event, detail) tuples:
 
       start         — a container transitions into running (or a genuinely new
                       container appears already running)
+      die           — also for a new container first seen already exited
+                      non-zero: a task that crashed between two samples (#5049)
+      restart       — Docker restarted it (its restart count went up); said
+                      instead of a start, since that is what happened
       stop          — running → not-running, or a running container is removed
       die           — same as stop but with a non-zero exit code (abnormal)
       oom           — OOMKilled flips False→True
@@ -62,14 +66,22 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
         old = old_state.get(name)
 
         if old is None:
-            # Newly observed container — only a start is meaningful (we have no
-            # prior state to diff a death against).
+            # Newly observed container: a start, or, when it is already down
+            # with a non-zero exit, the death that happened between samples.
             if new_running:
                 events.append((name, "start", c.get("image") or None))
+            elif new_status in _STOPPED_STATES and new_exit not in (None, 0):
+                events.append((name, "die", f"exit {new_exit}"))
             continue
 
         old_running = (old.get("status") or "").lower() == "running"
-        if new_running and not old_running:
+        new_restarts = c.get("restart_count") or 0
+        old_restarts = old.get("restart_count") or 0
+        if new_restarts > old_restarts:
+            gained = new_restarts - old_restarts
+            events.append((name, "restart", f"restart #{new_restarts}" if gained == 1
+                           else f"{gained} restarts, now #{new_restarts}"))
+        if new_running and not old_running and new_restarts <= old_restarts:
             events.append((name, "start", c.get("image") or None))
         elif old_running and not new_running:
             if new_exit not in (None, 0):
@@ -92,6 +104,55 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
     return events
 
 
+def _log_rows(log_batches, host_id: str):
+    """Pure: flatten the agent's log batches → docker_logs row kwargs (no DB).
+
+    `log_batches` is a list of (recorded_at, records) where each record is
+    {"container", "stream", "ts", "line"}. A line's own Docker timestamp wins;
+    recorded_at is the fallback when the agent couldn't parse one. Drops the
+    agent's advisory truncation marker (container "_steward" — it signals a
+    deferral, not a real container), malformed records, and lineless records;
+    normalises an unknown stream to stdout. Unit-testable in isolation.
+    """
+    for recorded_at, records in log_batches:
+        if not isinstance(records, list):
+            continue
+        for rec in records:
+            if not isinstance(rec, dict):
+                continue
+            name = rec.get("container")
+            line = rec.get("line")
+            if not name or name == "_steward" or line is None:
+                continue
+            ts = _parse_started_at(rec.get("ts")) or recorded_at
+            stream = rec.get("stream")
+            if stream not in ("stdout", "stderr"):
+                stream = "stdout"
+            yield {"host_id": host_id, "container_name": str(name)[:255],
+                   "ts": ts, "stream": stream, "line": str(line)}
+
+
+async def _persist_logs(session, host, log_batches) -> None:
+    """Append pushed container log lines (one row per line) for this host.
+
+    Time-series / append-only — the per-container size+age ring (retention)
+    bounds growth, so a chatty container just keeps a shorter window. The global
+    toggle + exclude list (Settings, no restart) are enforced here: this is the
+    authoritative drop point, since the push model has no channel to tell an
+    agent to stop collecting.
+    """
+    from steward.core.settings import get_setting
+    from .models import DockerLog
+
+    if not await get_setting(session, "docker.logs.enabled"):
+        return
+    exclude = set(await get_setting(session, "docker.logs.exclude") or ())
+    for row in _log_rows(log_batches, host.id):
+        if row["container_name"] in exclude:
+            continue
+        session.add(DockerLog(**row))
+
+
 async def _persist_swarm(session, host, swarm: dict) -> None:
     """Upsert this manager's swarm topology; drop rows no longer reported.
 
@@ -120,6 +181,7 @@ async def _persist_swarm(session, host, swarm: dict) -> None:
         row.running = int(s.get("running") or 0)
         row.image = s.get("image")
         row.placement_json = json.dumps(s.get("placement") or [])
+        row.tasks_json = json.dumps(s.get("tasks") or [])
         row.updated_at = now
     stale_services = delete(DockerSwarmService).where(
         DockerSwarmService.host_id == host.id)
@@ -201,7 +263,37 @@ async def _persist_disk(session, host, disk: dict) -> None:
     await session.execute(stale)
 
 
-async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -> None:
+async def _persist_status(session, host, status: dict) -> None:
+    """Record whether Docker answered on this host, and why not.
+
+    A host reporting "unavailable" with no row yet never had Docker: nothing to
+    record. Once a host has had Docker, losing it is recorded, so its last
+    containers read as stale rather than as running."""
+    from .models import DockerHostStatus
+
+    row = await session.get(DockerHostStatus, host.id)
+    if row is None:
+        if status["status"] == "unavailable":
+            return
+        row = DockerHostStatus(host_id=host.id)
+        session.add(row)
+    if row.status != status["status"]:
+        row.since = status["at"]
+    row.status = status["status"]
+    row.detail = (status.get("detail") or "")[:255] or None
+    row.reported_at = status["at"]
+
+
+async def _expire_swarm(session, host) -> None:
+    """Drop the swarm rows a host reported while it was a manager."""
+    from .models import DockerSwarmNode, DockerSwarmService
+
+    await session.execute(delete(DockerSwarmService).where(DockerSwarmService.host_id == host.id))
+    await session.execute(delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id))
+
+
+async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
+                              logs=None, status=None, phases=None) -> None:
     """Upsert containers + time-series + lifecycle events + swarm for one host.
 
     `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
@@ -211,15 +303,39 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -
     alert pipeline, and lifecycle-event derivation. `swarm` is the newest
     sample's swarm object (or None off managers) — persisted when present.
     `disk` is the newest sample's /system/df summary (or None on Docker-less
-    hosts) — persisted when present.
+    hosts) — persisted when present. `logs` is a list of (recorded_at, records)
+    log batches (m79) — appended one row per line when present, independent of
+    whether this sample also carried container metrics. `status` is the newest
+    sample's {"status", "detail", "at", "swarm_manager"} (agent ≥ 1.9.0): an
+    "ok" with an empty snapshot clears the host's containers, and a node that
+    says it is no longer a manager loses the swarm rows it reported as one.
+
+    `phases`, when given, is marked after each part (docker.status, .swarm,
+    .disk, .logs, .metrics, .events, .containers, .alerts, .reap), each
+    flushed before its mark, so a slow ingest names the part (#5181).
     """
-    from steward.core.alerts import record_metric
+    from steward.core.alerts import MetricBatch
     from .models import DockerContainer, DockerEvent, DockerMetric
 
+    async def mark(name: str) -> None:
+        if phases is not None:
+            await session.flush()
+            phases.mark(f"docker.{name}")
+
+    if status is not None:
+        await _persist_status(session, host, status)
+        if status.get("swarm_manager") is False:
+            await _expire_swarm(session, host)
+        await mark("status")
     if swarm is not None:
         await _persist_swarm(session, host, swarm)
+        await mark("swarm")
     if disk is not None:
         await _persist_disk(session, host, disk)
+        await mark("disk")
+    if logs:
+        await _persist_logs(session, host, logs)
+        await mark("logs")
 
     if not snapshots:
         return
@@ -238,6 +354,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -
                     mem_pct=c.get("mem_pct") or 0.0,
                     mem_usage_bytes=c.get("mem_usage_bytes") or 0,
                 ))
+    await mark("metrics")
 
     # Snapshot of stored per-container state BEFORE the upsert overwrites it —
     # used to diff lifecycle events. Skip event derivation on the host's first
@@ -247,17 +364,25 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -
     )).scalars().all()
     old_state = {
         r.name: {"status": r.status, "health": r.health,
-                 "oom_killed": r.oom_killed, "exit_code": r.exit_code}
+                 "oom_killed": r.oom_killed, "exit_code": r.exit_code,
+                 "restart_count": r.restart_count}
         for r in old_rows
     }
     if old_state:
+        # A stop is dated when Docker says it happened (agent ≥ 1.11), not when
+        # the sample that noticed it was taken.
+        finished = {c.get("name"): _parse_started_at(c.get("finished_at"))
+                    for c in latest_containers}
         for name, event, detail in _derive_events(old_state, latest_containers):
+            at = finished.get(name) if event in ("die", "stop") else None
             session.add(DockerEvent(
                 host_id=host.id, container_name=name,
-                event=event, detail=detail, at=latest_at,
+                event=event, detail=detail, at=at or latest_at,
             ))
+    await mark("events")
 
     # Current state + alerts from the newest snapshot only.
+    batch = MetricBatch()
     for c in latest_containers:
         name = c.get("name")
         if not name:
@@ -295,19 +420,19 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -
         # on different hosts don't collide in the metric/alert namespace.
         resource = f"{host.name}/{name}"
         if c.get("status") == "running" and c.get("cpu_pct") is not None:
-            await record_metric(
-                session=session, source_module="docker",
+            batch.add(
+                source_module="docker",
                 resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
             )
             if c.get("mem_pct") is not None:
-                await record_metric(
-                    session=session, source_module="docker",
+                batch.add(
+                    source_module="docker",
                     resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
                 )
         # Restart count is alertable regardless of state (crash-looping matters
         # most while the container is down/restarting).
-        await record_metric(
-            session=session, source_module="docker",
+        batch.add(
+            source_module="docker",
             resource_name=resource, metric_name="restart_count",
             value=float(c.get("restart_count", 0) or 0),
         )
@@ -316,12 +441,16 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -
         # container).
         health = c.get("health")
         if health in ("healthy", "unhealthy", "starting"):
-            await record_metric(
-                session=session, source_module="docker",
+            batch.add(
+                source_module="docker",
                 resource_name=resource, metric_name="is_healthy",
                 value=1.0 if health == "healthy" else 0.0,
             )
 
+    await mark("containers")
+    await batch.record(session)
+    await mark("alerts")
+
     # Reap containers that vanished from the host's latest listing. Without this,
     # removed one-shot containers (CI job runners, buildkit builders, codex jobs)
     # accumulate forever as permanent "stopped" rows and inflate every count.
@@ -334,3 +463,4 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -
     if seen_names:
         reap = reap.where(DockerContainer.name.notin_(seen_names))
     await session.execute(reap)
+    await mark("reap")

plugins/docker/mcp_tools.py

@@ -0,0 +1,642 @@
+"""The docker plugin's MCP tools and its get_overview findings.
+
+Registered through register_mcp_tools() in __init__.py, so they exist only
+while the plugin is enabled. Reads go through service.py, the same functions
+the plugin's pages use.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+
+from sqlalchemy import select
+
+from steward.ansible.runner import trigger_run
+from steward.ansible.sources import BUILTIN_SOURCE_NAME
+from steward.mcp._context import current_caller, mcp_app
+from steward.mcp.tools._audit import audit_mcp
+from steward.mcp.tools._shape import (
+    age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, summarize,
+)
+from steward.mcp.tools.overview import finding
+from steward.models.ansible_inventory import AnsibleTarget
+from steward.models.hosts import Host
+from steward.models.users import UserRole
+
+from . import service as docker_svc
+from .service import PROBLEM_SAYS, PROBLEM_SEVERITY, PROBLEMS, container_problems, problem_since
+from .dedup import dedup_by_container_id
+from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
+from .swarm_view import build_swarm_services, replaced_tasks, task_time
+
+CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
+LOG_LINE_CHARS = 2000
+
+
+# ── pure helpers ─────────────────────────────────────────────────────────────
+
+def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
+    """Keep both ends of an over-long line and say how much of the middle went."""
+    if text is None or len(text) <= limit:
+        return text
+    keep = limit // 2
+    return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
+
+
+def container_row(c, host_names: dict, problems: list[str], now: datetime,
+                  replaced: bool = False, since: datetime | None = None) -> dict:
+    running = (c.status or "").lower() == "running"
+    return {
+        "container_name": c.name,
+        "host_id": c.host_id,
+        "host_name": host_names.get(c.host_id),
+        "image": c.image,
+        "status": c.status,
+        "health": c.health,
+        "problems": problems,
+        # When the worst problem began, from the container's retained events.
+        "problem_since": iso(since),
+        "problem_age_s": age_s(since, now),
+        "exit_code": c.exit_code,
+        "oom_killed": bool(c.oom_killed),
+        "restart_count": c.restart_count,
+        "cpu_pct": rnd(c.cpu_pct),
+        "mem_pct": rnd(c.mem_pct),
+        "mem_usage_bytes": c.mem_usage_bytes,
+        "mem_limit_bytes": c.mem_limit_bytes,
+        "started_at": iso(c.started_at),
+        "uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
+        "compose_project": c.compose_project,
+        "swarm_service": c.service_name,
+        # A swarm task Swarm has replaced: history, judged on its service.
+        "swarm_replaced": replaced if c.service_name else None,
+        "reported_at": iso(c.scraped_at),
+        "reported_age_s": age_s(c.scraped_at, now),
+    }
+
+
+def _from_iso(text: str | None) -> datetime | None:
+    return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
+
+
+def container_findings(rows: list[dict]) -> list[dict]:
+    out = []
+    for r in rows:
+        if not r["problems"]:
+            continue
+        worst = r["problems"][0]
+        where = f" on {r['host_name']}" if r["host_name"] else ""
+        detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
+        out.append(finding(
+            PROBLEM_SEVERITY[worst], f"container_{worst}", r["container_name"],
+            f"container {r['container_name']}{where} {' and '.join(PROBLEM_SAYS[p] for p in r['problems'])}{detail}",
+            since=_from_iso(r["problem_since"]),
+            ids={"host_id": r["host_id"], "container_name": r["container_name"]},
+            next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
+    return out
+
+
+def task_error_text(err: dict | None) -> str:
+    """Swarm's own account of a task's end, as one clause."""
+    if not err:
+        return ""
+    why = err.get("error") or err.get("message") or err.get("state") or "unknown"
+    code = f", exit {err['exit_code']}" if err.get("exit_code") not in (None, 0) else ""
+    where = f" on {err['node']}" if err.get("node") else ""
+    return f"{why}{code}{where}"
+
+
+def service_findings(services: list[dict]) -> list[dict]:
+    """Swarm services that are failing: crash-looping, under-replicated, or that
+    lost a task to an error in the last 24h. Each names Swarm's last task error."""
+    out = []
+    for s in services:
+        replicas = f"{s['running']} of {s['desired']} replicas running"
+        last = task_error_text(s.get("last_error"))
+        tail = f"; last task error: {last}" if last else ""
+        ids = {"service_name": s["name"]}
+        since = s.get("trouble_since")
+        if s.get("crash_looping"):
+            out.append(finding(
+                "critical", "swarm_service_crash_looping", s["name"],
+                f"swarm service {s['name']} keeps failing: {s['failures_1h']} tasks failed in the "
+                f"last hour, {replicas}{tail}", since=since, ids=ids,
+                next="list_swarm_services()"))
+        elif s["state"] != "healthy":
+            out.append(finding(
+                "critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
+                s["name"], f"swarm service {s['name']} has {replicas}{tail}",
+                since=since, ids=ids, next="list_swarm_services()"))
+        elif s.get("failures_24h"):
+            n = s["failures_24h"]
+            out.append(finding(
+                "warning", "swarm_task_failed", s["name"],
+                f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
+                f"24h and is now {replicas}{tail}", since=since, ids=ids,
+                next="list_swarm_services()"))
+    return out
+
+
+# ── loading ──────────────────────────────────────────────────────────────────
+
+async def _host_names(db) -> dict[str, str]:
+    return dict((await db.execute(select(Host.id, Host.name))).all())
+
+
+def _crash_loop_deaths() -> int:
+    return docker_svc.crash_loop_deaths(mcp_app().config)
+
+
+async def _replaced(db, containers) -> set[tuple]:
+    services = list((await db.execute(select(DockerSwarmService))).scalars())
+    return replaced_tasks(containers, services)
+
+
+async def _rows(db, now: datetime) -> list[dict]:
+    """Every container with its problems. A swarm task Swarm replaced has none:
+    its failure, if it was one, belongs to its service (service_findings)."""
+    host_names = await _host_names(db)
+    return [container_row(r["container"], host_names, r["problems"], now,
+                          replaced=r["replaced"], since=r["since"])
+            for r in await docker_svc.ranked_containers(db, now, _crash_loop_deaths())]
+
+
+async def _swarm(db, now: datetime):
+    containers = dedup_by_container_id(await docker_svc.list_containers(db))
+    services = list((await db.execute(select(DockerSwarmService))).scalars())
+    nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
+    host_names = await _host_names(db)
+    node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
+    built = build_swarm_services(containers, services, node_hostname, host_names,
+                                 now=now, crash_loop_deaths=_crash_loop_deaths())
+    return built["services"], nodes, host_names
+
+
+async def mcp_overview(db) -> list[dict]:
+    """get_overview findings: failing containers and under-replicated swarm services."""
+    now = now_utc()
+    rows = await _rows(db, now)
+    services, _, _ = await _swarm(db, now)
+    return container_findings(rows) + service_findings(services)
+
+
+# ── tools ────────────────────────────────────────────────────────────────────
+
+async def list_containers(host: str = "", status: str = "", problem: str = "",
+                          compose_project: str = "", limit: int = 100) -> dict:
+    """Docker containers across every host that runs a Steward agent with Docker.
+
+    host: a host's name or id (list_hosts); empty for all hosts.
+    status: Docker's state — "running", "exited", "restarting", "paused",
+      "created" or "dead"; empty for all.
+    problem: "any" for containers with something wrong, or one of
+      "restarting", "crash_looping" (died repeatedly within the last hour;
+      the count is set under Settings → Thresholds, default 3), "unhealthy"
+      (failing its healthcheck), "oom_killed", "exited_nonzero" (died with an
+      error in the last 24h; exit 137/143 without an OOM kill is a stop, not
+      an error); empty for no filter.
+    compose_project: only containers from this Compose project.
+    limit: default 100, max 500.
+
+    Containers with problems come first. Each row's `problems` lists what is
+    wrong, most severe first, and `problem_since` / `problem_age_s` say when
+    the worst of them began (null when Steward's retained events cannot say).
+    `reported_age_s` is how old the agent's last report is, so a large value
+    means the row may be stale.
+
+    A swarm task's container that Swarm has replaced (a rolling update, or a
+    retry after a failure) has `swarm_replaced: true` and no problems: Swarm
+    restarts a task as a new container, so failures are counted on the
+    service — list_swarm_services shows them.
+    """
+    if problem and problem != "any" and problem not in PROBLEMS:
+        raise ValueError(f"problem {problem!r} is not 'any' or one of {', '.join(PROBLEMS)}.")
+    limit = clamp_limit(limit, 100, 500)
+    now = now_utc()
+    async with db_session() as db:
+        rows = await _rows(db, now)
+        if host:
+            hosts = list((await db.execute(select(Host))).scalars())
+            h = pick(host, hosts, kind="host", list_tool="list_hosts")
+            rows = [r for r in rows if r["host_id"] == h.id]
+    if status:
+        rows = [r for r in rows if (r["status"] or "").lower() == status.lower()]
+    if problem == "any":
+        rows = [r for r in rows if r["problems"]]
+    elif problem:
+        rows = [r for r in rows if problem in r["problems"]]
+    if compose_project:
+        rows = [r for r in rows if r["compose_project"] == compose_project]
+    rows.sort(key=lambda r: (not r["problems"], (r["status"] or "") != "running",
+                             (r["host_name"] or "").lower(), r["container_name"]))
+    return bounded(rows, limit, key="containers",
+                   more="Pass a larger `limit` (max 500), or filter by `host`, `status` or `problem`.")
+
+
+@dataclass
+class RemovedContainer:
+    """A container Docker has removed, known only by its retained logs and events."""
+    host_id: str
+    name: str
+    last_seen: datetime
+
+
+_REMOVED_HINT = ("It may have been removed: Swarm replaces a dead task with a new container under "
+                 "a new name. list_containers(status=\"exited\") lists stopped containers, and "
+                 "search_container_logs(host=...) searches a whole host.")
+
+
+async def _pick_container(db, container: str, host: str, *, removed: bool = False):
+    """The container named `container`: a live one, else (with `removed`) one
+    Docker has removed whose logs or events Steward still holds, returned as a
+    RemovedContainer. Exact names beat partial ones, and live beats removed."""
+    containers = await docker_svc.list_containers(db)
+    host_id = None
+    if host:
+        hosts = list((await db.execute(select(Host))).scalars())
+        host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
+        containers = [c for c in containers if c.host_id == host_id]
+    want = (container or "").strip()
+    if not want:
+        raise ValueError("pass a container name; list_containers lists them.")
+    live = {(c.host_id, c.name) for c in containers}
+
+    def _one(found, what):
+        if len(found) > 1:
+            raise ValueError(f"{len(found)} hosts have {what} named {want!r}; pass `host` "
+                             f"(one of: {', '.join(sorted(f[0] for f in found))}).")
+        return found[0] if found else None
+
+    same = [c for c in containers if c.name == want] or \
+           [c for c in containers if c.name.lower() == want.lower()]
+    if same:
+        return _one([(c.host_id, c) for c in same], "a container")[1]
+    if removed:
+        gone = [r for r in await docker_svc.retained_names(db, want, exact=True, host_id=host_id)
+                if (r[0], r[1]) not in live]
+        if gone:
+            return RemovedContainer(*_one(gone, "a removed container"))
+    # One more chance: a unique container whose name contains it (swarm task
+    # names look like "stack_svc.1.k2j…").
+    partial = [c for c in containers if want.lower() in c.name.lower()]
+    if len(partial) == 1:
+        return partial[0]
+    gone = []
+    if removed and not partial:
+        gone = [r for r in await docker_svc.retained_names(db, want, exact=False, host_id=host_id)
+                if (r[0], r[1]) not in live]
+        if len(gone) == 1:
+            return RemovedContainer(*gone[0])
+    names = sorted(c.name for c in partial)[:10] + [f"{r[1]} (removed)" for r in gone[:10]]
+    hint = f" Containers whose names contain it: {', '.join(names)}." if names else ""
+    raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} "
+                     f"{'' if names else _REMOVED_HINT + ' '}list_containers lists them.")
+
+
+def _removed_note(c: RemovedContainer) -> str:
+    return (f"{c.name} no longer exists: Docker removed it (Swarm replaces a failed task with a new "
+            f"container). This is what Steward retained; it was last seen {iso(c.last_seen)}.")
+
+
+async def get_container(container: str, host: str = "", range: str = "24h", events: int = 20) -> dict:
+    """One container in depth: state, resources over time and lifecycle events.
+
+    container: the container's name, as list_containers shows. A unique part of
+      a name also works (swarm task names are long).
+    host: the host's name or id. Needed only when several hosts run a
+      container with this name.
+    range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
+      max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
+      "7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
+    events: how many lifecycle events (start, restart, stop, die, oom, health_change)
+      to return, newest first (default 20, max 100).
+
+    A container Docker has since removed (a dead Swarm task, a `docker rm`)
+    still resolves by its exact name, or by a part of it that matches one
+    removed container, while Steward retains its logs or events: the reply
+    has `removed: true`, `last_seen_at`, its history and its events, and no
+    live state. A live container has `removed: false`.
+    """
+    since, key = lookback(range)
+    events = clamp_limit(events, 20, 100)
+    now = now_utc()
+    async with db_session() as db:
+        c = await _pick_container(db, container, host, removed=True)
+        if isinstance(c, RemovedContainer):
+            return await _removed_container(db, c, since, key, events, now)
+        gone = (c.host_id, c.name) in await _replaced(db, await docker_svc.list_containers(db))
+        failures = (await docker_svc.recent_failures_by_container(
+            db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
+        host_names = await _host_names(db)
+        cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
+        mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
+        evs = await docker_svc.container_events(db, c.host_id, c.name, events)
+        sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)])
+    hour_ago = now - timedelta(hours=1)
+    recent = sum(1 for e in failures if e.at >= hour_ago)
+    try:
+        ports = json.loads(c.ports_json or "[]")
+    except ValueError:
+        ports = []
+    problems = [] if gone else container_problems(c, recent, bool(failures), _crash_loop_deaths())
+    began = problem_since(problems, failures, hour_ago, sick.get((c.host_id, c.name)))
+    return {
+        **container_row(c, host_names, problems, now, replaced=gone, since=began),
+        "removed": False,
+        "container_id": c.container_id,
+        "ports": ports,
+        "net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
+        "blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
+        "range": key,
+        "cpu_pct_history": summarize(cpu),
+        "mem_pct_history": summarize(mem),
+        "events": _event_rows(evs, now),
+        "next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
+    }
+
+
+def _event_rows(evs, now: datetime) -> list[dict]:
+    return [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)}
+            for e in evs]
+
+
+async def _removed_container(db, c: RemovedContainer, since, key: str, events: int,
+                             now: datetime) -> dict:
+    """get_container for a container Docker has removed: what Steward kept."""
+    host_names = await _host_names(db)
+    cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
+    mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
+    evs = await docker_svc.container_events(db, c.host_id, c.name, events)
+    return {
+        "container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id),
+        "removed": True, "note": _removed_note(c),
+        "last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now),
+        "range": key,
+        "cpu_pct_history": summarize(cpu),
+        "mem_pct_history": summarize(mem),
+        "events": _event_rows(evs, now),
+        "next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")',
+    }
+
+
+async def list_swarm_services(limit: int = 100) -> dict:
+    """Docker Swarm services with their replicas per node, and the swarm's nodes.
+
+    `state` is "healthy" (all desired replicas running), "degraded" (some) or
+    "down" (none). `replicas` are the service's current tasks: each names its
+    host and container status, and one marked `ghost` runs on a node with no
+    Steward agent, so Steward knows only how many tasks are placed there.
+    Tasks Swarm replaced (rolling updates, retries) are left out and counted
+    in `previous_tasks`.
+
+    From Swarm's own task records (a manager's agent 1.8.0 or later;
+    `tasks_known: false` and nulls otherwise):
+    - `failures_1h` / `failures_24h`: tasks that failed or were rejected.
+      Swarm retries a failed task as a new one, so this, not any container's
+      restart count, is how a crash loop shows. Swarm keeps about 5 old tasks
+      per replica, which bounds the count.
+    - `crash_looping`: failures_1h reached the crash-loop count set under
+      Settings → Thresholds (default 3).
+    - `last_error`: the newest task that failed, was rejected, or carries an
+      error while Swarm retries it — `error` is Swarm's own text (e.g. "No
+      such image", "task: non-zero exit (1)"), with `exit_code`, `node`, `at`.
+    - `trouble_since` / `trouble_age_s`: when the current trouble began — a
+      crash loop's first failure in the hour, a short service's first failed
+      task in its failing slots, a recovered service's first failure in 24h.
+      Swarm keeps few old tasks, so a long outage reads as a lower bound.
+
+    Services with a problem come first. `nodes` lists every swarm node once,
+    with its role (manager/worker), availability and status.
+    limit: default 100, max 500.
+    """
+    limit = clamp_limit(limit, 100, 500)
+    now = now_utc()
+    async with db_session() as db:
+        services, nodes, host_names = await _swarm(db, now)
+    seen, node_rows = set(), []
+    for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
+        if n.node_id in seen:
+            continue
+        seen.add(n.node_id)
+        node_rows.append({"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
+                          "leader": n.leader, "availability": n.availability, "status": n.status,
+                          "reported_by": host_names.get(n.host_id)})
+    node_rows.sort(key=lambda n: n["hostname"].lower())
+    order = {"down": 0, "degraded": 1, "healthy": 2}
+
+    def _rank(s):
+        return (not s["crash_looping"], order.get(s["state"], 3), not s["failures_24h"], s["name"])
+
+    def _error_row(err):
+        if err is None:
+            return None
+        at = task_time(err)
+        return {"error": err["error"], "message": err["message"], "state": err["state"],
+                "exit_code": err["exit_code"], "node": err["node"],
+                "at": iso(at), "age_s": age_s(at, now) if at else None}
+
+    rows = [
+        {
+            "service_name": s["name"], "mode": s["mode"], "state": s["state"],
+            "running": s["running"], "desired": s["desired"],
+            "crash_looping": s["crash_looping"],
+            "failures_1h": s["failures_1h"], "failures_24h": s["failures_24h"],
+            "last_error": _error_row(s["last_error"]),
+            "trouble_since": iso(s["trouble_since"]),
+            "trouble_age_s": age_s(s["trouble_since"], now),
+            "previous_tasks": s["previous_tasks"], "tasks_known": s["tasks_known"],
+            "replicas": [
+                {"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
+                {"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
+                 "status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
+                 "cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
+                for r in s["replicas"]
+            ],
+        }
+        for s in sorted(services, key=_rank)
+    ]
+    return {"nodes": node_rows, **bounded(rows, limit, key="services")}
+
+
+async def search_container_logs(container: str = "", host: str = "", text: str = "",
+                                range: str = "1h", stream: str = "", before: str = "",
+                                context: int = 0, limit: int = 50) -> dict:
+    """Search the container log lines Steward has retained, newest first.
+
+    container: one container's name (as list_containers shows); empty searches
+      every container (narrow with `host`). A container Docker has removed
+      still works while its lines are retained: the reply then has
+      `container_removed: true` and a `note` saying when it was last seen.
+    host: a host's name or id; empty for all hosts.
+    text: case-insensitive substring to match; empty matches every line.
+    range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are
+      kept per container up to a size and age limit, so older lines may be gone.
+    stream: "stdout" or "stderr"; empty for both.
+    before: an ISO time; returns only lines older than it. Pass a response's
+      `next_before` to page further back.
+    context: lines of surrounding log (0-5) to include with each match, from
+      the same container. Only with `container` set.
+    limit: matches to return, default 50, max 200.
+
+    Lines longer than 2000 characters keep both ends, and the middle is
+    replaced by a marker saying how much was cut.
+    """
+    since, key = lookback(range)
+    limit = clamp_limit(limit, 50, 200)
+    context = max(0, min(int(context or 0), 5))
+    if context and not container:
+        raise ValueError("context needs `container`: it shows lines around a match within one container's log.")
+    if stream and stream not in ("stdout", "stderr"):
+        raise ValueError(f"stream {stream!r} is not 'stdout' or 'stderr'.")
+    before_dt = None
+    if before:
+        try:
+            before_dt = datetime.fromisoformat(before.replace("Z", "+00:00"))
+        except ValueError as exc:
+            raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc
+    async with db_session() as db:
+        host_id = name = removed = None
+        if container:
+            c = await _pick_container(db, container, host, removed=True)
+            host_id, name = c.host_id, c.name
+            removed = c if isinstance(c, RemovedContainer) else None
+        elif host:
+            hosts = list((await db.execute(select(Host))).scalars())
+            host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
+        rows = await docker_svc.search_logs(db, since=since, host_id=host_id, name=name, text=text,
+                                            stream=stream, before=before_dt, limit=limit + 1)
+        more = len(rows) > limit
+        rows = rows[:limit]
+        host_names = await _host_names(db)
+        matches = []
+        for r in rows:
+            m = {"at": iso(r.ts), "container_name": r.container_name, "host_id": r.host_id,
+                 "host_name": host_names.get(r.host_id), "stream": r.stream, "line": cut_middle(r.line)}
+            if context:
+                pre, post = await docker_svc.log_context(db, r.host_id, r.container_name, r.ts, context)
+                m["before"] = [cut_middle(x.line) for x in pre]
+                m["after"] = [cut_middle(x.line) for x in post]
+            matches.append(m)
+    out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more}
+    if removed is not None:
+        out["container_removed"] = True
+        out["note"] = _removed_note(removed)
+        if removed.last_seen < since:
+            out["note"] += " That is before this range; pass a longer `range`."
+    if more:
+        # Full precision: `at` is cut to the second for reading, and a cursor
+        # cut the same way would skip lines logged earlier in that second.
+        out["next_before"] = rows[-1].ts.isoformat()
+        out["more"] = "More matches are older; pass next_before as `before`, or narrow with `text`."
+    return out
+
+
+async def get_docker_disk_usage(host: str = "") -> dict:
+    """Docker disk usage per host: images, containers, volumes and build cache, in bytes.
+
+    host: a host's name or id; empty for every host. `images_reclaimable_bytes`
+    is what pruning unused images would free. An admin can prune from
+    Steward's Docker → Disk page.
+    """
+    async with db_session() as db:
+        summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
+        host_names = await _host_names(db)
+        if host:
+            hosts = list((await db.execute(select(Host))).scalars())
+            hid = pick(host, hosts, kind="host", list_tool="list_hosts").id
+            summaries = [s for s in summaries if s.host_id == hid]
+    rows = [
+        {
+            "host_id": s.host_id, "host_name": host_names.get(s.host_id),
+            "images_total": s.images_total, "images_active": s.images_active,
+            "images_bytes": s.images_size, "images_reclaimable_bytes": s.images_reclaimable,
+            "containers_count": s.containers_count, "containers_bytes": s.containers_size,
+            "volumes_count": s.volumes_count, "volumes_bytes": s.volumes_size,
+            "build_cache_bytes": s.build_cache_size,
+            "reported_at": iso(s.updated_at), "reported_age_s": age_s(s.updated_at),
+        }
+        for s in summaries
+    ]
+    rows.sort(key=lambda r: -(r["images_reclaimable_bytes"] or 0))
+    return {"hosts": rows, "total": len(rows)}
+
+
+PRUNE_BUILTIN_PLAYBOOK = "maintenance/docker_prune.yml"
+_STOPPED = ("exited", "created", "dead")
+
+
+def prune_preview(what: str, usage, stopped: int) -> dict:
+    """What a prune of `what` would remove, from the host's latest disk report."""
+    out: dict = {}
+    if what in ("containers", "system"):
+        out["stopped_containers"] = stopped
+    if what == "images":
+        out["unused_images_bytes"] = usage.images_reclaimable if usage else None
+    if what == "system":
+        out["dangling_images"] = "yes (unused but tagged images stay)"
+        out["build_cache_bytes"] = usage.build_cache_size if usage else None
+        out["unused_networks"] = "yes"
+    return out
+
+
+async def prune_docker(host: str, what: str, check: bool = True) -> dict:
+    """Reclaim Docker disk on a host. Previews unless check=false.
+
+    host: the Steward host's name or id. It needs a linked Ansible target
+      (list_inventory shows the links); the prune runs through Ansible as
+      Steward's built-in docker_prune playbook.
+    what: "containers" removes stopped containers; "images" removes ALL
+      unused images, not only dangling ones; "system" is `docker system
+      prune -f`: stopped containers, dangling images, unused networks and
+      build cache. Volumes are never pruned.
+    check: true (default) starts nothing and returns `would_remove`, sized
+      from the host's latest Docker disk report. Ansible's own check mode
+      skips the prune commands, so a check-mode run would show nothing.
+      Pass false to run the prune.
+
+    Admin only, as on the Docker → Disk page. A real prune returns `run_id`
+    (follow it with wait_for_run) and is recorded in the audit log.
+    """
+    what = (what or "").strip()
+    if what not in docker_svc.PRUNE_TARGETS:
+        raise ValueError(f"what {what!r} is not one of {', '.join(docker_svc.PRUNE_TARGETS)}.")
+    async with db_session() as db:
+        h = pick(host, (await db.execute(select(Host))).scalars(), kind="host", list_tool="list_hosts")
+        target = (await db.execute(
+            select(AnsibleTarget).where(AnsibleTarget.host_id == h.id))).scalars().first()
+        usage = await db.get(DockerDiskUsage, h.id)
+        stopped = sum(1 for c in await docker_svc.list_containers(db, h.id)
+                      if (c.status or "").lower() in _STOPPED)
+    if target is None:
+        raise ValueError(f"host {h.name!r} has no linked Ansible target, so Steward cannot reach it "
+                         "to prune. Link one under Ansible → Inventory. Nothing was run.")
+    preview = {
+        "host": h.name,
+        "what": what,
+        "ansible_target": target.name,
+        "would_remove": prune_preview(what, usage, stopped),
+        "disk_report_age_s": age_s(usage.updated_at) if usage else None,
+    }
+    if check:
+        return {**preview, "check": True,
+                "summary": f"Nothing was run. Pass check=false to prune {what} on {h.name}."}
+    caller = current_caller()
+    run, _src, err = await trigger_run(
+        mcp_app(), source_name=BUILTIN_SOURCE_NAME, playbook_path=PRUNE_BUILTIN_PLAYBOOK,
+        inventory_scope=f"steward:target:{target.id}",
+        params={"extra_vars_map": docker_svc.prune_extra_vars(what)},
+        triggered_by=caller.user_id, origin="mcp")
+    if err:
+        raise ValueError(f"the prune could not start: {err}.")
+    await audit_mcp("docker.prune.started", "host", h.name,
+                    {"host_id": h.id, "what": what, "run_id": run.id})
+    return {**preview, "check": False, "run_id": run.id,
+            "next": f'wait_for_run(run_id="{run.id}")'}
+
+
+def register_tools(register) -> None:
+    for fn in (list_containers, get_container, list_swarm_services,
+               search_container_logs, get_docker_disk_usage):
+        register(fn, role=UserRole.viewer, read_only=True)
+    register(prune_docker, role=UserRole.admin, read_only=False)

plugins/docker/migrations/versions/docker_009_container_logs.py

@@ -0,0 +1,48 @@
+"""Docker container logs table
+
+Adds docker_logs — one row per container log line, pushed by the host agent and
+folded into its metrics push. Time-series, host-scoped (container names are only
+unique within a host). Bounded by a per-container size+age ring in retention, so
+a chatty container keeps a shorter window rather than growing without limit.
+Additive create_table + twin indexes (viewer lookup by (host, container, ts);
+age-cutoff prune by ts).
+
+Revision ID: docker_009_container_logs
+Revises: docker_008_bigint_mem
+Create Date: 2026-07-19
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_009_container_logs"
+down_revision: Union[str, None] = "docker_008_bigint_mem"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "docker_logs",
+        sa.Column("id", sa.String(length=36), nullable=False),
+        sa.Column("host_id", sa.String(length=36), nullable=False),
+        sa.Column("container_name", sa.String(length=255), nullable=False),
+        sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("stream", sa.String(length=8), nullable=False, server_default="stdout"),
+        sa.Column("line", sa.Text(), nullable=False, server_default=""),
+        sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index("ix_docker_logs_host_id", "docker_logs", ["host_id"])
+    op.create_index("ix_docker_logs_container_name", "docker_logs", ["container_name"])
+    op.create_index("ix_docker_logs_host_container_time",
+                    "docker_logs", ["host_id", "container_name", "ts"])
+    op.create_index("ix_docker_logs_ts", "docker_logs", ["ts"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_docker_logs_ts", table_name="docker_logs")
+    op.drop_index("ix_docker_logs_host_container_time", table_name="docker_logs")
+    op.drop_index("ix_docker_logs_container_name", table_name="docker_logs")
+    op.drop_index("ix_docker_logs_host_id", table_name="docker_logs")
+    op.drop_table("docker_logs")

plugins/docker/migrations/versions/docker_010_swarm_tasks.py

@@ -0,0 +1,30 @@
+"""Docker swarm service tasks column
+
+Adds docker_swarm_services.tasks_json — each service's newest tasks as a
+manager's agent (≥ 1.8.0) reports them: slot, node, desired and current state,
+Swarm's error and message, exit code and time. Lets Steward tell a task Swarm
+replaced from one that crashed, count a service's failures across task
+retries, and say why its last task failed. Additive column; no DROP+recreate.
+
+Revision ID: docker_010_swarm_tasks
+Revises: docker_009_container_logs
+Create Date: 2026-10-06
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_010_swarm_tasks"
+down_revision: Union[str, None] = "docker_009_container_logs"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("docker_swarm_services",
+                  sa.Column("tasks_json", sa.Text, nullable=False,
+                            server_default="[]"))
+
+
+def downgrade() -> None:
+    op.drop_column("docker_swarm_services", "tasks_json")

plugins/docker/migrations/versions/docker_011_host_status.py

@@ -0,0 +1,36 @@
+"""Docker host status
+
+Adds docker_host_status: per host, whether Docker answered at the agent's
+last report (ok / unavailable / error) and why not. Lets Steward show a host
+whose Docker went away as such, instead of its last containers as running
+(#5152). Additive create_table.
+
+Revision ID: docker_011_host_status
+Revises: docker_010_swarm_tasks
+Create Date: 2026-10-06
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_011_host_status"
+down_revision: Union[str, None] = "docker_010_swarm_tasks"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "docker_host_status",
+        sa.Column("host_id", sa.String(length=36), nullable=False),
+        sa.Column("status", sa.String(length=16), nullable=False),
+        sa.Column("detail", sa.String(length=255), nullable=True),
+        sa.Column("since", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("reported_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("host_id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("docker_host_status")

plugins/docker/models.py

@@ -1,6 +1,5 @@
 # plugins/docker/models.py
 from __future__ import annotations
-import uuid
 from datetime import datetime, timezone
 from sqlalchemy import (
     BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text,
@@ -8,6 +7,7 @@ from sqlalchemy import (
 )
 from sqlalchemy.orm import Mapped, mapped_column
 from steward.models.base import Base
+from steward.core.ids import time_ordered_id
 
 
 class DockerContainer(Base):
@@ -67,7 +67,7 @@ class DockerMetric(Base):
     __tablename__ = "docker_metrics"
 
     id: Mapped[str] = mapped_column(
-        String(36), primary_key=True, default=lambda: str(uuid.uuid4())
+        String(36), primary_key=True, default=time_ordered_id
     )
     host_id: Mapped[str] = mapped_column(
         String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
@@ -103,7 +103,7 @@ class DockerMetricHourly(Base):
     __tablename__ = "docker_metrics_hourly"
 
     id: Mapped[str] = mapped_column(
-        String(36), primary_key=True, default=lambda: str(uuid.uuid4())
+        String(36), primary_key=True, default=time_ordered_id
     )
     host_id: Mapped[str] = mapped_column(
         String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
@@ -139,14 +139,14 @@ class DockerEvent(Base):
     __tablename__ = "docker_events"
 
     id: Mapped[str] = mapped_column(
-        String(36), primary_key=True, default=lambda: str(uuid.uuid4())
+        String(36), primary_key=True, default=time_ordered_id
     )
     host_id: Mapped[str] = mapped_column(
         String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
     )
     container_name: Mapped[str] = mapped_column(String(255), nullable=False)
     event: Mapped[str] = mapped_column(String(16), nullable=False)
-    # start | stop | die | oom | health_change
+    # start | restart | stop | die | oom | health_change
     detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
     # e.g. "exit 137", "healthy→unhealthy", image on start
     at: Mapped[datetime] = mapped_column(
@@ -163,6 +163,43 @@ class DockerEvent(Base):
     )
 
 
+class DockerLog(Base):
+    """Container log lines pushed by the host agent — one row per line.
+
+    Time-series, scoped to the reporting host (container names are only unique
+    within a host, same identity as docker_metrics). The agent tails each running
+    container incrementally and folds new lines into its metrics push; `ts` is the
+    line's own Docker timestamp. Bounded by a per-container size+age ring
+    (retention), so a chatty container just keeps a shorter window rather than
+    growing without limit.
+    """
+    __tablename__ = "docker_logs"
+
+    id: Mapped[str] = mapped_column(
+        String(36), primary_key=True, default=time_ordered_id
+    )
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
+    ts: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False,
+        default=lambda: datetime.now(timezone.utc),
+    )
+    stream: Mapped[str] = mapped_column(String(8), nullable=False, default="stdout")
+    # stdout | stderr
+    line: Mapped[str] = mapped_column(Text, nullable=False, default="")
+
+    # Viewer filters on (host_id, container_name) and sorts by time; the ring
+    # prune walks the same key newest-first. A second index on `ts` alone serves
+    # the age-cutoff delete. Twin-index idiom, mirroring docker_events.
+    __table_args__ = (
+        Index("ix_docker_logs_host_container_time",
+              "host_id", "container_name", "ts"),
+        Index("ix_docker_logs_ts", "ts"),
+    )
+
+
 class DockerSwarmService(Base):
     """A Swarm service as seen by a manager host: desired vs running replicas.
 
@@ -186,6 +223,11 @@ class DockerSwarmService(Base):
     # A manager sees every task, so this captures cross-node placement that the
     # local docker_containers rows (this host only) can't.
     placement_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
+    # The service's newest tasks, newest first, as the manager's agent shaped
+    # them: [{"task_id", "slot", "node_id", "desired_state", "state", "message",
+    # "error", "exit_code", "at"}], JSON. Swarm's own record of which tasks it
+    # replaced and why each one stopped; "[]" from agents older than 1.8.0.
+    tasks_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
     updated_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False,
         default=lambda: datetime.now(timezone.utc),
@@ -262,3 +304,24 @@ class DockerSwarmNode(Base):
         DateTime(timezone=True), nullable=False,
         default=lambda: datetime.now(timezone.utc),
     )
+
+
+class DockerHostStatus(Base):
+    """Whether Docker answered on a host at its agent's last report (agent ≥ 1.9.0).
+
+    "ok" when it did; "unavailable" when there is no Docker to ask (no socket,
+    nothing listening); "error" when Docker is there and the call failed.
+    `detail` says why for the last two. While a host is not "ok" its container
+    rows are what Docker last said, not what is true now, and read as stale.
+    A host that never had Docker gets no row.
+    """
+    __tablename__ = "docker_host_status"
+
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
+    )
+    status: Mapped[str] = mapped_column(String(16), nullable=False)
+    detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    # When the status last changed, and the agent's newest report.
+    since: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    reported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)

plugins/docker/overview.py

@@ -0,0 +1,150 @@
+# plugins/docker/overview.py
+"""The Docker page's two tables, as plain rows (#5299, #5301): one row per
+swarm service and one per standalone container, with the search, stack, host
+and status filters and the column sort the page offers. Model-free, so it is
+unit-tested without a database; routes.rows() feeds it.
+
+A stack is how the operator thinks of a group of services. Swarm names a
+stack's services "<stack>_<service>", so a service's stack is the part of its
+name before the first "_"; a standalone container's is its compose project.
+Stacks filter the list rather than section it, as host groups do on Hosts.
+"""
+from __future__ import annotations
+
+from datetime import datetime
+
+from .swarm_view import service_trouble
+
+# Status filter values and their labels; NO_STACK picks rows with no stack.
+DOCKER_STATES = {"problem": "Needs attention", "healthy": "Healthy", "stopped": "Stopped"}
+NO_STACK = "__none__"
+# Each sort and whether it starts descending: the busiest first; "up" starts
+# with the shortest uptime, the thing that most recently restarted.
+DOCKER_SORTS = {"status": False, "name": False, "stack": False, "cpu": True, "mem": True,
+                "up": False}
+
+
+def stack_of(service_name: str) -> str | None:
+    """The stack a swarm service was deployed in: "arr" for "arr_radarr";
+    None for a name with no "_" (a service created on its own)."""
+    stack, sep, rest = service_name.partition("_")
+    return stack if sep and stack and rest else None
+
+
+def _seconds_up(started_at, now: datetime) -> float | None:
+    return None if started_at is None else max((now - started_at).total_seconds(), 0.0)
+
+
+def service_row(s: dict, now: datetime) -> dict:
+    """A built swarm service (swarm_view.build_swarm_services) as a table row.
+    CPU and memory add up the running replicas; "up" is the newest replica's
+    start, so a service that just restarted a task reads as just restarted."""
+    reals = [r for r in s["replicas"] if not r["ghost"]]
+    running = [r for r in reals if (r["status"] or "").lower() == "running"]
+    cpu = [r["cpu_pct"] for r in running if r.get("cpu_pct") is not None]
+    mem = [r["mem_bytes"] for r in running if r.get("mem_bytes") is not None]
+    starts = [r["started_at"] for r in running if r.get("started_at")]
+    started = max(starts) if starts else None
+    return {
+        "kind": "service", "name": s["name"], "stack": stack_of(s["name"]),
+        "mode": s["mode"],
+        "image": s.get("image") or next((r["image"] for r in reals if r.get("image")), None),
+        "running": s["running"], "desired": s["desired"], "state": s["state"],
+        "trouble": service_trouble(s),
+        "nodes": sorted({r["host"] for r in reals}, key=str.lower),
+        "ghost_nodes": sorted({r["host"] for r in s["replicas"] if r["ghost"]}, key=str.lower),
+        "cpu_cores": sum(cpu) / 100 if cpu else None,
+        "mem_bytes": sum(mem) if mem else None,
+        "started_at": started, "up_s": _seconds_up(started, now),
+        "unhealthy": any(r.get("health") == "unhealthy" for r in reals),
+        "failures_24h": s.get("failures_24h"), "crash_looping": s.get("crash_looping"),
+        "last_error": s.get("last_error"), "trouble_since": s.get("trouble_since"),
+    }
+
+
+def container_trouble(c, *, stale: bool, exit_failed: bool) -> int | None:
+    """How bad a standalone container is, on service_trouble's scale: 1
+    unhealthy or restarting, 2 died with a failing exit, 3 its report is
+    stale; None when nothing is wrong (a cleanly stopped one included)."""
+    if c.health == "unhealthy" or c.status == "restarting":
+        return 1
+    if c.status != "running" and exit_failed:
+        return 2
+    if stale:
+        return 3
+    return None
+
+
+def container_row(c, *, label: str, host: str, usage: dict, ports: list, stale: bool,
+                  exit_failed: bool, now: datetime) -> dict:
+    """A standalone container as a table row; `usage` is service.usage()."""
+    running = c.status == "running"
+    return {
+        "kind": "container", "name": label, "container_name": c.name, "host": host,
+        "host_id": c.host_id, "stack": c.compose_project or None, "image": c.image,
+        "status": c.status, "health": c.health, "running": running, "ports": ports,
+        "usage": usage, "cpu_cores": usage.get("cores") if running else None,
+        "mem_bytes": usage.get("mem_bytes") if running else None,
+        "started_at": c.started_at if running else None,
+        "up_s": _seconds_up(c.started_at, now) if running else None,
+        "exit_code": c.exit_code, "oom_killed": c.oom_killed, "exit_failed": exit_failed,
+        "restart_count": c.restart_count or 0, "stale": stale, "scraped_at": c.scraped_at,
+        "trouble": container_trouble(c, stale=stale, exit_failed=exit_failed),
+    }
+
+
+def _hosts(row: dict) -> list[str]:
+    return row["nodes"] + row["ghost_nodes"] if row["kind"] == "service" else [row["host"]]
+
+
+def _stopped(row: dict) -> bool:
+    return (row["running"] == 0) if row["kind"] == "service" else not row["running"]
+
+
+def filter_rows(rows: list[dict], *, q: str = "", stack: str = "", host: str = "",
+                state: str = "") -> list[dict]:
+    """Rows matching a search (name, container name, image or stack, any
+    case), a stack (or NO_STACK), a host or node, and a Status filter; an
+    empty value doesn't filter."""
+    q = (q or "").strip().lower()
+    out = []
+    for r in rows:
+        if q and not any(q in (v or "").lower() for v in
+                         (r["name"], r.get("container_name"), r["image"], r["stack"])):
+            continue
+        if (stack == NO_STACK and r["stack"]) or (stack and stack != NO_STACK and r["stack"] != stack):
+            continue
+        if host and host not in _hosts(r):
+            continue
+        if state == "problem" and r["trouble"] is None:
+            continue
+        if state == "healthy" and (r["trouble"] is not None or _stopped(r)):
+            continue
+        if state == "stopped" and not _stopped(r):
+            continue
+        out.append(r)
+    return out
+
+
+def sort_rows(rows: list[dict], sort: str, desc: bool) -> list[dict]:
+    """Rows in a column's order. Ties keep name order; a row with no value for
+    the column goes last whichever way it runs. "status" is problems first,
+    worst first, then running, then stopped."""
+    by_name = sorted(rows, key=lambda r: r["name"].lower())
+    if sort == "name":
+        return list(reversed(by_name)) if desc else by_name
+    if sort == "status":
+        return sorted(by_name, key=lambda r: (4 if r["trouble"] is None else r["trouble"],
+                                              _stopped(r)), reverse=desc)
+    value = {"stack": lambda r: (r["stack"] or "").lower() or None,
+             "cpu": lambda r: r["cpu_cores"], "mem": lambda r: r["mem_bytes"],
+             "up": lambda r: r["up_s"]}[sort]
+    have = [r for r in by_name if value(r) is not None]
+    return sorted(have, key=value, reverse=desc) + [r for r in by_name if value(r) is None]
+
+
+def filter_options(services: list[dict], containers: list[dict]) -> dict:
+    """The stack and host choices the filter bar offers, from every row."""
+    rows = services + containers
+    return {"stacks": sorted({r["stack"] for r in rows if r["stack"]}, key=str.lower),
+            "hosts": sorted({h for r in rows for h in _hosts(r)}, key=str.lower)}

plugins/docker/retention.py

@@ -38,6 +38,8 @@ async def run_docker_retention(
     events_days: int,
     metrics_raw_days: int,
     metrics_rollup_days: int,
+    logs_retention_days: int = 3,
+    logs_max_bytes_per_container: int = 5_000_000,
     now: datetime | None = None,
 ) -> dict:
     """Roll up + prune Docker time-series. Returns a counts dict for logging.
@@ -47,18 +49,22 @@ async def run_docker_retention(
        re-run is idempotent, then delete those raw rows.
     2. Prune rolled-up rows older than the rollup window.
     3. Prune docker_events older than the events window.
+    4. Prune docker_logs with a per-container size+age ring (m79): drop lines
+       older than the age window, then keep only the newest ~cap bytes per
+       (host, container).
     """
     from datetime import timezone
 
     from sqlalchemy import delete, func, select
     from sqlalchemy.dialects.postgresql import insert as pg_insert
 
-    from .models import DockerEvent, DockerMetric, DockerMetricHourly
+    from .models import DockerEvent, DockerLog, DockerMetric, DockerMetricHourly
 
     if now is None:
         now = datetime.now(timezone.utc)
 
     rolled = rolled_rows = events_pruned = rollup_pruned = 0
+    logs_age_pruned = logs_size_pruned = 0
 
     # ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
     raw_cutoff = _rollup_cutoff(now, metrics_raw_days)
@@ -120,9 +126,35 @@ async def run_docker_retention(
     )
     events_pruned = res.rowcount or 0
 
+    # ── 4. Container-log ring: age cutoff, then per-container byte cap (m79) ──
+    logs_cutoff = now - timedelta(days=logs_retention_days)
+    res = await session.execute(
+        delete(DockerLog).where(DockerLog.ts < logs_cutoff)
+    )
+    logs_age_pruned = res.rowcount or 0
+
+    # Size ring: per (host, container), sum line bytes newest-first; delete a row
+    # once its strictly-newer siblings already fill the cap. Using the EXCLUSIVE
+    # prefix (running total minus this row) means the newest row always survives,
+    # so a single line larger than the cap is never wiped out.
+    running = func.sum(func.length(DockerLog.line)).over(
+        partition_by=[DockerLog.host_id, DockerLog.container_name],
+        order_by=[DockerLog.ts.desc(), DockerLog.id.desc()],
+    )
+    prefix_excl = (running - func.length(DockerLog.line)).label("prefix_excl")
+    ranked = select(DockerLog.id, prefix_excl).subquery()
+    over_cap = select(ranked.c.id).where(
+        ranked.c.prefix_excl >= logs_max_bytes_per_container)
+    res = await session.execute(
+        delete(DockerLog).where(DockerLog.id.in_(over_cap))
+    )
+    logs_size_pruned = res.rowcount or 0
+
     return {
         "buckets_rolled": rolled,
         "raw_rows_rolled": rolled_rows,
         "rollup_pruned": rollup_pruned,
         "events_pruned": events_pruned,
+        "logs_age_pruned": logs_age_pruned,
+        "logs_size_pruned": logs_size_pruned,
     }

plugins/docker/service.py

@@ -0,0 +1,685 @@
+"""Docker queries shared by the plugin's pages and its MCP tools.
+
+Each function takes an open session and returns rows or plain data, with no
+Quart request context, so the routes and the /mcp tools (which run outside
+Quart's pipeline) call the same code.
+"""
+from __future__ import annotations
+
+import re
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import func, select
+
+from steward.core.time_range import bucket_seconds
+from steward.models.hosts import Host
+
+from .dedup import dedup_by_container_id
+from .models import (
+    DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly, DockerSwarmService,
+)
+from .swarm_view import replaced_tasks
+
+
+CRASH_LOOP_DEATHS = 3  # default; Settings → Thresholds sets it
+
+
+def crash_loop_deaths(config) -> int:
+    """Deaths within an hour that make a container or a swarm service a crash
+    loop, from Settings → Thresholds (app.config["THRESHOLDS"])."""
+    th = (config.get("THRESHOLDS") or {}).get("crash_loop") or {}
+    return int(th.get("warn") or CRASH_LOOP_DEATHS)
+
+
+# Exit codes a deliberate stop leaves: SIGTERM (143), and SIGKILL (137) when
+# the process outlives the stop timeout. Without an OOM kill they are someone
+# stopping the container, not the container failing.
+STOP_EXITS = (137, 143)
+
+
+def exit_is_failure(exit_code, oom_killed: bool = False) -> bool:
+    """Whether a container's exit is a failure: an OOM kill, or a non-zero
+    exit other than a stop's."""
+    return bool(oom_killed) or exit_code not in (None, 0, *STOP_EXITS)
+
+
+def _event_exit(detail) -> int | None:
+    """The exit code in a die event's detail ("exit 1"), or None."""
+    text = (detail or "").strip()
+    if text.startswith("exit "):
+        try:
+            return int(text[5:])
+        except ValueError:
+            return None
+    return None
+
+
+def event_is_failure(e) -> bool:
+    """Whether a die/oom event is a failure. An oom always is; a die is unless
+    its exit is a stop's (or clean)."""
+    if e.event == "oom":
+        return True
+    return e.event == "die" and exit_is_failure(_event_exit(e.detail))
+
+
+def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
+    """What failed in the last 24h, newest first, for the dashboard count and
+    the Docker page's list of it: one row per plain container that died or was
+    OOM-killed, and one per swarm service with failed tasks.
+
+    `events` are die/oom events of the last 24h. A swarm task's container dies
+    whenever Swarm replaces it, so a service with Swarm's task records counts
+    its failed tasks instead of its containers' deaths; a service without them
+    (an agent older than 1.8.0) falls back to its tasks' deaths.
+
+    Each row: {kind: "container"|"service", name, host_id, host, count, at,
+    detail, task_id}; task_id is the failed swarm task, so its logs can be
+    found (None for a container, or a service without task records).
+    """
+    by_prefix = {s["name"] + ".": s for s in swarm_services}
+    rows: dict[tuple, dict] = {}
+    # Each manager reports every swarm task, so one task's death can arrive
+    # once per manager: a service counts each dead task container once.
+    task_deaths: dict[tuple, set] = {}
+    for e in sorted(events, key=lambda e: e.at, reverse=True):
+        if not event_is_failure(e):
+            continue
+        svc = next((s for p, s in by_prefix.items() if e.container_name.startswith(p)), None)
+        if svc is not None and svc["tasks_known"]:
+            continue
+        kind, name = ("service", svc["name"]) if svc else ("container", e.container_name)
+        key = (kind, name) if svc else (kind, e.host_id, name)
+        if svc is not None:
+            seen = task_deaths.setdefault(key, set())
+            if e.container_name in seen:
+                continue
+            seen.add(e.container_name)
+        row = rows.get(key)
+        if row is None:
+            rows[key] = {"kind": kind, "name": name, "host_id": e.host_id,
+                         "host": host_name.get(e.host_id, e.host_id), "count": 1, "at": e.at,
+                         "detail": "OOM-killed" if e.event == "oom" else e.detail,
+                         "task_id": (task_of(svc["name"], e.container_name) or (None, None))[1]
+                         if svc else None}
+        else:
+            row["count"] += 1
+    from .swarm_view import task_time
+    for s in swarm_services:
+        if s["tasks_known"] and s["failures_24h"]:
+            err = s["last_error"] or {}
+            rows[("service", s["name"])] = {
+                "kind": "service", "name": s["name"], "host_id": None, "host": err.get("node"),
+                "count": s["failures_24h"], "at": task_time(err) if err else None,
+                "detail": err.get("error") or err.get("message") or err.get("state"),
+                "task_id": err.get("task_id")}
+    epoch = datetime.min.replace(tzinfo=timezone.utc)
+    return sorted(rows.values(), key=lambda r: r["at"] or epoch, reverse=True)
+
+
+def task_label(c, node_hostname: dict | None = None) -> str:
+    """How a container reads to a person. A swarm task is its service and
+    slot: "codex_db · 1" for a replica, "agent · node-3" for a global
+    service's task on a node. Anything else as display_name reads it."""
+    from .swarm_view import _slot_key
+    svc = getattr(c, "service_name", None)
+    if not svc:
+        return display_name(c)
+    slot = _slot_key(c)
+    if slot.isdigit():
+        return f"{svc} · {slot}"
+    host = (node_hostname or {}).get(slot) or (node_hostname or {}).get(getattr(c, "node_id", None))
+    return f"{svc} · {host}" if host else svc
+
+
+_CI_JOB = re.compile(
+    r"^(?:GITEA|FORGEJO)-ACTIONS-TASK-\d+-WORKFLOW-(?P<workflow>.+?)-JOB-(?P<job>.+?)(?:-[0-9a-f]{16,})?$")
+# docker buildx's container driver names its node "buildx_buildkit_<builder><n>".
+_BUILDKIT = re.compile(r"^buildx_buildkit_(?P<builder>.+?)\d*$")
+
+
+def display_name(c) -> str:
+    """A container's name as a list can show it in one short line (#5253): a
+    swarm task by its service (the host says where it runs), a Gitea or
+    Forgejo Actions job container as "CI: <workflow> · <job>", a buildx
+    builder as "BuildKit builder · <name>", anything else by its own name."""
+    svc = getattr(c, "service_name", None)
+    if svc:
+        return svc
+    m = _CI_JOB.match(c.name or "")
+    if m:
+        return f"CI: {m['workflow']} · {m['job']}"
+    m = _BUILDKIT.match(c.name or "")
+    if m:
+        return f"BuildKit builder · {m['builder']}"
+    return c.name
+
+
+def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None) -> dict:
+    """A container's CPU and memory in terms a reader can weigh.
+
+    Docker's CPU% is per core (100% = one core busy), so it is shown as cores
+    and graded as a share of the host's CPUs when the host's core count is
+    known. Docker's memory limit is the host's RAM when none is set, so memory
+    is shown in bytes, against a limit only when one is actually set; mem_pct
+    is then a share of that limit, otherwise of the host's memory. `cpu_pct`
+    replaces the container's last sample (an average, say) when given.
+    """
+    cpu = getattr(c, "cpu_pct", None) if cpu_pct is None else cpu_pct
+    limit = getattr(c, "mem_limit_bytes", None)
+    limited = bool(limit and mem_total and limit < mem_total * 0.95)
+    return {
+        "cores": None if cpu is None else cpu / 100.0,
+        "cpu_share": None if cpu is None or not cpus else cpu / cpus,
+        "cpus": cpus,
+        "mem_bytes": getattr(c, "mem_usage_bytes", None),
+        "mem_limit": limit if limited else None,
+        "mem_pct": getattr(c, "mem_pct", None),
+        "mem_of": "its limit" if limited else ("the host's memory" if mem_total else None),
+    }
+
+
+async def host_capacity(db, host_ids) -> dict[str, tuple[int | None, int | None]]:
+    """{host_id: (CPU cores, memory bytes)} from the host agent's latest
+    readings: cores counted from its per-core CPU series, memory from
+    available ÷ (1 − used%). (None, None) for a host without an agent."""
+    from sqlalchemy import or_
+    from steward.models.metrics import PluginMetricLatest
+    hosts = await host_map(db, set(host_ids))
+    if not hosts:
+        return {}
+    names = {h.name: hid for hid, h in hosts.items()}
+    rows = (await db.execute(
+        select(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name,
+               PluginMetricLatest.value)
+        .where(PluginMetricLatest.source_module == "host_agent",
+               or_(PluginMetricLatest.resource_name.in_(list(names)),
+                   *(PluginMetricLatest.resource_name.like(escape_like(n) + ":core%", escape="\\")
+                     for n in names)))
+    )).all()
+    cores: dict[str, int] = {}
+    mem: dict[str, dict] = {}
+    for res, metric, value in rows:
+        if res in names:
+            mem.setdefault(res, {})[metric] = value
+        elif metric == "cpu_pct":
+            host = res.rsplit(":core", 1)[0]
+            cores[host] = cores.get(host, 0) + 1
+    out = {}
+    for name, hid in names.items():
+        m = mem.get(name, {})
+        avail, used = m.get("mem_available_bytes"), m.get("mem_used_pct")
+        total = int(avail / (1 - used / 100)) if avail and used is not None and used < 99.5 else None
+        out[hid] = (cores.get(name), total)
+    return out
+
+
+def bucket(values: list[float], target: int = 40) -> list[float]:
+    """Bucket-average a series down to ~target points (DB-agnostic, in Python).
+
+    Replaces the old SQL strftime() bucketing, which was SQLite-only and broke
+    on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of
+    rows — averaging into a readable point count keeps the sparkline's shape.
+    """
+    n = len(values)
+    if n <= target:
+        return values
+    size = (n + target - 1) // target
+    out: list[float] = []
+    for i in range(0, n, size):
+        chunk = values[i:i + size]
+        out.append(sum(chunk) / len(chunk))
+    return out
+
+
+async def container_history(db, host_id: str, name: str, since) -> tuple[list, list]:
+    """Return (cpu_series, mem_series) sparkline-ready for one host's container."""
+    rows = (await db.execute(
+        select(DockerMetric.cpu_pct, DockerMetric.mem_pct)
+        .where(DockerMetric.host_id == host_id)
+        .where(DockerMetric.container_name == name)
+        .where(DockerMetric.scraped_at >= since)
+        .order_by(DockerMetric.scraped_at)
+    )).all()
+    cpu = bucket([r.cpu_pct or 0.0 for r in rows])
+    mem = bucket([r.mem_pct or 0.0 for r in rows])
+    return cpu, mem
+
+
+async def host_map(db, host_ids: set[str]) -> dict[str, Host]:
+    if not host_ids:
+        return {}
+    return {
+        h.id: h for h in (await db.execute(
+            select(Host).where(Host.id.in_(host_ids)))).scalars().all()
+    }
+
+
+def escape_like(text: str) -> str:
+    """Make `text` match literally inside a LIKE/ILIKE pattern (escape char \\)."""
+    return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+
+
+async def list_containers(db, host_id: str | None = None) -> list[DockerContainer]:
+    """Containers, running first then by name; one host's when host_id is given."""
+    stmt = select(DockerContainer)
+    if host_id is not None:
+        stmt = stmt.where(DockerContainer.host_id == host_id)
+    stmt = stmt.order_by((DockerContainer.status == "running").desc(), DockerContainer.name)
+    return list((await db.execute(stmt)).scalars())
+
+
+async def container_events(db, host_id: str, name: str, limit: int = 50) -> list[DockerEvent]:
+    """A container's lifecycle events (start/stop/die/oom/health), newest first."""
+    return list((await db.execute(
+        select(DockerEvent)
+        .where(DockerEvent.host_id == host_id)
+        .where(DockerEvent.container_name == name)
+        .order_by(DockerEvent.at.desc())
+        .limit(limit)
+    )).scalars())
+
+
+async def query_logs(db, host_id: str, name: str, stream: str, query: str,
+                     limit: int) -> list:
+    """The recent retained lines for one container, newest-first, optionally
+    filtered by stream and a case-insensitive substring.
+
+    Newest-first is deliberate: the viewer replaces the list on each poll, which
+    resets scroll to the top — so the latest lines stay visible without any
+    scroll handling, and older lines are a scroll away.
+    """
+    stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line)
+            .where(DockerLog.host_id == host_id)
+            .where(DockerLog.container_name == name))
+    if stream in ("stdout", "stderr"):
+        stmt = stmt.where(DockerLog.stream == stream)
+    if query:
+        stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\"))
+    stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
+    return (await db.execute(stmt)).all()
+
+
+CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes")
+
+
+async def container_points(db, host_id: str, name: str, metric: str, since,
+                           target: int = 120) -> list[tuple]:
+    """(time, average) points of one container metric since `since`, oldest first.
+
+    Raw samples are bucket-averaged in SQL; the part of the range older than
+    the oldest raw sample (already rolled up and pruned by retention) comes
+    from the hourly table.
+    """
+    if metric not in CONTAINER_SERIES:
+        raise ValueError(f"metric must be one of {', '.join(CONTAINER_SERIES)}")
+    raw_col = getattr(DockerMetric, metric)
+    width_s = bucket_seconds(since, target)
+    rbucket = func.date_bin(
+        func.make_interval(0, 0, 0, 0, 0, 0, width_s), DockerMetric.scraped_at, func.to_timestamp(0),
+    ).label("bucket")
+    raw = (await db.execute(
+        select(rbucket, func.avg(raw_col))
+        .where(DockerMetric.host_id == host_id, DockerMetric.container_name == name,
+               DockerMetric.scraped_at >= since)
+        .group_by(rbucket).order_by(rbucket)
+    )).all()
+    oldest_raw = raw[0][0] if raw else None
+    hourly_stmt = (
+        select(DockerMetricHourly.bucket, getattr(DockerMetricHourly, metric))
+        .where(DockerMetricHourly.host_id == host_id,
+               DockerMetricHourly.container_name == name,
+               DockerMetricHourly.bucket >= since)
+        .order_by(DockerMetricHourly.bucket)
+    )
+    if oldest_raw is not None:
+        hourly_stmt = hourly_stmt.where(DockerMetricHourly.bucket < oldest_raw)
+    hourly = (await db.execute(hourly_stmt)).all()
+    return [(t, float(v)) for t, v in hourly] + [(t, float(v)) for t, v in raw]
+
+
+# Busiest Containers bars (#5214): the recent past in BARS equal slices.
+BARS = 24
+
+
+def bar_slots(now: datetime, hours: int, bars: int = BARS) -> tuple[list[datetime], int]:
+    """The start of each of the last `bars` buckets covering `hours`, oldest
+    first, aligned as date_bin aligns them (to the epoch), and their width in
+    seconds. The newest bucket is the one `now` falls in."""
+    width = max(60, hours * 3600 // bars)
+    last = int(now.timestamp()) // width * width
+    return [datetime.fromtimestamp(last - (bars - 1 - i) * width, timezone.utc)
+            for i in range(bars)], width
+
+
+async def recent_usage(db, keys, since: datetime, width: int) -> dict:
+    """{(host_id, container_name): {bucket start: (cpu_pct, mem_usage_bytes)}}
+    averaged per bucket since `since`, for the given containers, in one query."""
+    keys = set(keys)
+    if not keys:
+        return {}
+    bucket = func.date_bin(
+        func.make_interval(0, 0, 0, 0, 0, 0, int(width)), DockerMetric.scraped_at, func.to_timestamp(0),
+    ).label("bucket")
+    rows = (await db.execute(
+        select(DockerMetric.host_id, DockerMetric.container_name, bucket,
+               func.avg(DockerMetric.cpu_pct), func.avg(DockerMetric.mem_usage_bytes))
+        .where(DockerMetric.scraped_at >= since,
+               DockerMetric.host_id.in_({h for h, _ in keys}),
+               DockerMetric.container_name.in_({n for _, n in keys}))
+        .group_by(DockerMetric.host_id, DockerMetric.container_name, bucket)
+    )).all()
+    out: dict = {}
+    for host_id, name, at, cpu, mem in rows:
+        if (host_id, name) in keys:
+            out.setdefault((host_id, name), {})[at] = (
+                None if cpu is None else float(cpu), None if mem is None else float(mem))
+    return out
+
+
+def scaled_bars(values: list, floor: float) -> dict:
+    """Bar heights in % of a roof: the row's own peak, but never below `floor`,
+    so a container idling at 0.05 cores draws flat and its 0.05 → 0.10 wobble
+    doesn't look like a 1.5 → 3 core spike. None (no samples) stays a gap."""
+    present = [v for v in values if v is not None]
+    peak = max(present) if present else None
+    roof = max(peak or 0.0, floor)
+    return {"bars": [None if v is None else round(100.0 * v / roof, 1) for v in values],
+            "peak": peak, "roof": roof}
+
+
+async def search_logs(db, *, since, host_id: str | None = None, name: str | None = None,
+                      text: str = "", stream: str = "", before=None, limit: int = 50) -> list:
+    """Retained log lines matching a filter, newest first (limit + 1 fetched by callers
+    that need to know whether more exist)."""
+    stmt = (select(DockerLog.id, DockerLog.host_id, DockerLog.container_name,
+                   DockerLog.ts, DockerLog.stream, DockerLog.line)
+            .where(DockerLog.ts >= since))
+    if host_id:
+        stmt = stmt.where(DockerLog.host_id == host_id)
+    if name:
+        stmt = stmt.where(DockerLog.container_name == name)
+    if stream in ("stdout", "stderr"):
+        stmt = stmt.where(DockerLog.stream == stream)
+    if text:
+        stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(text)}%", escape="\\"))
+    if before is not None:
+        stmt = stmt.where(DockerLog.ts < before)
+    stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
+    return list((await db.execute(stmt)).all())
+
+
+async def log_context(db, host_id: str, name: str, ts, n: int) -> tuple[list, list]:
+    """Up to n lines either side of `ts` in one container's log, each oldest first."""
+    cols = (DockerLog.id, DockerLog.ts, DockerLog.stream, DockerLog.line)
+    base = select(*cols).where(DockerLog.host_id == host_id, DockerLog.container_name == name)
+    before = (await db.execute(
+        base.where(DockerLog.ts < ts).order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(n)
+    )).all()
+    after = (await db.execute(
+        base.where(DockerLog.ts > ts).order_by(DockerLog.ts, DockerLog.id).limit(n)
+    )).all()
+    return list(reversed(before)), list(after)
+
+
+async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]:
+    """{(host_id, container_name): [die/oom events since `since`, newest first]},
+    leaving out a die that was a stop (event_is_failure)."""
+    rows = (await db.execute(
+        select(DockerEvent)
+        .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
+        .order_by(DockerEvent.at.desc())
+    )).scalars()
+    out: dict[tuple[str, str], list] = {}
+    for e in rows:
+        if event_is_failure(e):
+            out.setdefault((e.host_id, e.container_name), []).append(e)
+    return out
+
+
+async def failure_events(db, since) -> list:
+    """Die and OOM events since `since`, newest first; failures_24h() decides
+    which of them are failures."""
+    return list((await db.execute(
+        select(DockerEvent)
+        .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
+        .order_by(DockerEvent.at.desc())
+    )).scalars())
+
+
+async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None,
+                         limit: int = 20) -> list[tuple[str, str, datetime]]:
+    """(host_id, container_name, last seen) of containers Steward still holds
+    logs or events for, newest first — including ones Docker has removed.
+
+    exact: the name itself, from both logs and events. Otherwise names that
+    contain `want` (case-insensitive), from events only: a substring match
+    over every retained log line's name would scan the log table, and a
+    container Steward watched die has a die or stop event.
+    """
+    seen: dict[tuple[str, str], datetime] = {}
+    sources = ((DockerLog, DockerLog.ts), (DockerEvent, DockerEvent.at)) if exact \
+        else ((DockerEvent, DockerEvent.at),)
+    for model, at in sources:
+        match = (model.container_name == want if exact
+                 else model.container_name.ilike(f"%{escape_like(want)}%", escape="\\"))
+        stmt = (select(model.host_id, model.container_name, func.max(at))
+                .where(match).group_by(model.host_id, model.container_name))
+        if host_id:
+            stmt = stmt.where(model.host_id == host_id)
+        for hid, name, last in (await db.execute(stmt)).all():
+            key = (hid, name)
+            if key not in seen or last > seen[key]:
+                seen[key] = last
+    rows = sorted(((h, n, t) for (h, n), t in seen.items()), key=lambda r: r[2], reverse=True)
+    return rows[:limit]
+
+
+# A swarm task's container is named "<service>.<slot>.<task id>": slot is the
+# replica number, or the node id for a global service; task ids are 25 chars.
+_TASK_SUFFIX = re.compile(r"^(?P<slot>[A-Za-z0-9]+)\.(?P<task>[a-z0-9]{25})$")
+
+
+def task_of(service: str, container_name: str) -> tuple[str, str] | None:
+    """(slot, task id) when `container_name` is one of `service`'s task
+    containers, else None — so "arr" does not claim "arr_radarr.1.…"."""
+    if not container_name.startswith(service + "."):
+        return None
+    m = _TASK_SUFFIX.match(container_name[len(service) + 1:])
+    return (m["slot"], m["task"]) if m else None
+
+
+async def service_containers(db, service: str) -> list[tuple[str, str]]:
+    """(host_id, container_name) of every task container of a swarm service
+    that Steward still holds anything for: the live ones, and the ones Docker
+    removed that left events (each task container has a start event). The
+    page's logs and history read across all of them (#5302)."""
+    pattern = escape_like(service + ".") + "%"
+    found: set[tuple[str, str]] = set()
+    for model, col in ((DockerContainer, DockerContainer.name),
+                       (DockerEvent, DockerEvent.container_name)):
+        rows = (await db.execute(
+            select(model.host_id, col).where(col.like(pattern, escape="\\")).distinct())).all()
+        found.update((h, n) for h, n in rows if task_of(service, n))
+    return sorted(found)
+
+
+async def query_service_logs(db, containers, stream: str, query: str, limit: int,
+                             task: str | None = None) -> list:
+    """The newest retained lines across a service's task containers, newest
+    first, each with its container name so the page can say which task wrote
+    it. `task` keeps one task's lines (a failure's "Logs →")."""
+    from sqlalchemy import tuple_
+    pairs = [(h, n) for h, n in containers if task is None or n.endswith("." + task)]
+    if not pairs:
+        return []
+    stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line, DockerLog.container_name)
+            .where(tuple_(DockerLog.host_id, DockerLog.container_name).in_(pairs)))
+    if stream in ("stdout", "stderr"):
+        stmt = stmt.where(DockerLog.stream == stream)
+    if query:
+        stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\"))
+    stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
+    return (await db.execute(stmt)).all()
+
+
+def sum_series(samples, since: datetime, until: datetime, points: int = 60) -> list:
+    """[(bucket start, cpu %, memory bytes)] summed across containers: each
+    container's samples are averaged within a bucket, then the containers are
+    added, so two replicas read as their combined load. `samples` are
+    (container_name, at, cpu_pct, mem_bytes); buckets with no sample are left
+    out rather than read as zero."""
+    width = max((until - since).total_seconds() / points, 60.0)
+    acc: dict[int, dict[str, list]] = {}
+    for name, at, cpu, mem in samples:
+        i = int((at - since).total_seconds() // width)
+        if i < 0:
+            continue
+        per = acc.setdefault(i, {}).setdefault(name, [0.0, 0, 0])
+        per[0] += cpu or 0.0
+        per[1] += mem or 0
+        per[2] += 1
+    out = []
+    for i in sorted(acc):
+        cpu = sum(c / n for c, _, n in acc[i].values())
+        mem = sum(m / n for _, m, n in acc[i].values())
+        out.append((since + timedelta(seconds=i * width), cpu, int(mem)))
+    return out
+
+
+async def service_history(db, containers, since: datetime) -> list:
+    """sum_series over a service's task containers' samples since `since`."""
+    from sqlalchemy import tuple_
+    if not containers:
+        return []
+    rows = (await db.execute(
+        select(DockerMetric.container_name, DockerMetric.scraped_at,
+               DockerMetric.cpu_pct, DockerMetric.mem_usage_bytes)
+        .where(tuple_(DockerMetric.host_id, DockerMetric.container_name).in_(list(containers)),
+               DockerMetric.scraped_at >= since)
+        .order_by(DockerMetric.scraped_at))).all()
+    return sum_series(rows, since, datetime.now(timezone.utc))
+
+
+async def unhealthy_since(db, keys) -> dict[tuple[str, str], datetime]:
+    """{(host_id, container_name): when it last turned unhealthy} for the given
+    containers, from their health_change events; absent when none is retained."""
+    keys = set(keys)
+    if not keys:
+        return {}
+    rows = (await db.execute(
+        select(DockerEvent)
+        .where(DockerEvent.event == "health_change",
+               DockerEvent.detail.like("%→unhealthy"),
+               DockerEvent.container_name.in_({name for _, name in keys}))
+        .order_by(DockerEvent.at.desc())
+    )).scalars()
+    out: dict[tuple[str, str], datetime] = {}
+    for e in rows:
+        key = (e.host_id, e.container_name)
+        if key in keys and key not in out:
+            out[key] = e.at
+    return out
+
+
+PRUNE_TARGETS = ("containers", "images", "system")
+
+
+def prune_extra_vars(prune_target: str) -> dict:
+    """Extra-vars for the bundled docker_prune playbook. "Prune unused images"
+    means ALL unused (docker image prune -a), not just dangling; the full system
+    prune stays conservative (-f, no -a) per the operator's choice (m78)."""
+    extra_vars: dict = {"prune_target": prune_target}
+    if prune_target == "images":
+        extra_vars["prune_all_images"] = True
+    return extra_vars
+
+
+# ── what is wrong with a container ───────────────────────────────────────────
+# Shared by the Docker page's "Needs attention" and the MCP tools, so both
+# rank the same containers the same way.
+
+PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
+
+def container_problems(c, failures_1h: int, failed_24h: bool,
+                       crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
+    """What is wrong with one container, most severe first; [] when nothing is."""
+    status = (c.status or "").lower()
+    out = []
+    if status == "restarting":
+        out.append("restarting")
+    if failures_1h >= crash_loop_deaths:
+        out.append("crash_looping")
+    if status == "running" and (c.health or "") == "unhealthy":
+        out.append("unhealthy")
+    if c.oom_killed:
+        out.append("oom_killed")
+    if status in ("exited", "dead") and exit_is_failure(c.exit_code) and failed_24h:
+        out.append("exited_nonzero")
+    return out
+
+
+def problem_since(problems: list[str], deaths: list, hour_ago: datetime,
+                  unhealthy_at: datetime | None = None) -> datetime | None:
+    """When a container's worst problem began; None when the retained events
+    cannot say. `deaths` are its die/oom events of the last 24h, newest first.
+
+    A restart or crash loop dates from its first death in the last hour, an
+    OOM kill or error exit from the death that ended it, and a failing
+    healthcheck from when its health last turned unhealthy.
+    """
+    if not problems:
+        return None
+    worst = problems[0]
+    if worst in ("restarting", "crash_looping"):
+        in_hour = [e.at for e in deaths if e.at >= hour_ago]
+        return min(in_hour) if in_hour else (deaths[0].at if deaths else None)
+    if worst == "unhealthy":
+        return unhealthy_at
+    if worst == "oom_killed":
+        ooms = [e.at for e in deaths if e.event == "oom"]
+        return ooms[0] if ooms else (deaths[0].at if deaths else None)
+    return deaths[0].at if deaths else None
+
+
+PROBLEM_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
+             "oom_killed": "warning", "exited_nonzero": "warning"}
+PROBLEM_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
+         "unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
+         "exited_nonzero": "exited with an error"}
+
+
+
+async def ranked_containers(db, now: datetime, crash_loop_deaths: int) -> list[dict]:
+    """Every container (deduplicated) with what is wrong with it, most severe
+    first: {container, problems, since, replaced}. A swarm task Swarm has
+    replaced has no problems; its failure belongs to its service."""
+    containers = dedup_by_container_id(await list_containers(db))
+    services = list((await db.execute(select(DockerSwarmService))).scalars())
+    replaced = replaced_tasks(containers, services)
+    failures = await recent_failures_by_container(db, now - timedelta(hours=24))
+    hour_ago = now - timedelta(hours=1)
+    sick = await unhealthy_since(
+        db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"])
+    out = []
+    for c in containers:
+        key = (c.host_id, c.name)
+        gone = key in replaced
+        events = failures.get(key, [])
+        recent = sum(1 for e in events if e.at >= hour_ago)
+        problems = [] if gone else container_problems(c, recent, bool(events), crash_loop_deaths)
+        out.append({"container": c, "problems": problems, "replaced": gone,
+                    "since": problem_since(problems, events, hour_ago, sick.get(key))})
+    return out
+
+
+def needs_attention(ranked: list[dict]) -> list[dict]:
+    """The containers with something wrong, worst problem first, then by host
+    and name."""
+    return sorted((r for r in ranked if r["problems"]),
+                  key=lambda r: (PROBLEMS.index(r["problems"][0]), r["container"].host_id, r["container"].name))
+
+
+def stale_after_seconds(config) -> int:
+    """How old a container's last report may be before the page calls it
+    stale: the host agent's own staleness window, since the agent reports it."""
+    return int(((config.get("PLUGINS") or {}).get("host_agent") or {}).get("stale_after_seconds", 180))

plugins/docker/swarm_nodes.py

@@ -0,0 +1,71 @@
+# plugins/docker/swarm_nodes.py
+"""The Swarm page laid out by node (#5303): one card per node with its role,
+whether it takes work, what its containers use against what the machine has,
+and the services Swarm placed on it. Model-free, so it is unit-tested without
+a database; routes.swarm() feeds it.
+
+A node with a Steward agent reports its own containers, so its services carry
+CPU and memory. A node without one is known only from the managers' placement:
+its services are listed by count, with no figures.
+"""
+from __future__ import annotations
+
+from .swarm_view import _placement
+
+
+def node_ok(n) -> bool:
+    """A node that is up and taking work."""
+    return n.status == "ready" and n.availability == "active"
+
+
+def node_cards(nodes, services, containers, *, node_host: dict, host_names: dict,
+               capacity: dict, service_state: dict) -> list[dict]:
+    """One card per node, a node in trouble first, then managers, then by name.
+
+    `nodes` and `services` are one swarm's deduplicated rows; `containers`
+    every container row (only running swarm tasks on these nodes count);
+    `node_host` maps a node id to the Steward host on it, `capacity` a host id
+    to (cores, memory bytes), `service_state` a service name to healthy,
+    degraded or down."""
+    running_on: dict[str, dict[str, dict]] = {}
+    for c in containers:
+        if not (c.node_id and c.service_name and c.status == "running"):
+            continue
+        row = running_on.setdefault(c.node_id, {}).setdefault(
+            c.service_name, {"name": c.service_name, "tasks": 0, "cpu_pct": 0.0, "mem_bytes": 0,
+                             "measured": True})
+        row["tasks"] += 1
+        row["cpu_pct"] += c.cpu_pct or 0.0
+        row["mem_bytes"] += c.mem_usage_bytes or 0
+    placed: dict[str, dict[str, int]] = {}
+    for s in services:
+        for p in _placement(s):
+            count = int(p.get("running") or 0)
+            if count > 0 and p.get("node_id"):
+                placed.setdefault(p["node_id"], {})[s.service_name] = count
+
+    cards = []
+    for n in nodes:
+        rows = dict(running_on.get(n.node_id, {}))
+        for name, count in placed.get(n.node_id, {}).items():
+            rows.setdefault(name, {"name": name, "tasks": count, "cpu_pct": None, "mem_bytes": None,
+                                   "measured": False})
+        for r in rows.values():
+            r["state"] = service_state.get(r["name"], "healthy")
+            r["cores"] = r["cpu_pct"] / 100 if r["cpu_pct"] is not None else None
+        tasks = sorted(rows.values(), key=lambda r: (r["state"] == "healthy", -(r["cores"] or 0.0),
+                                                     r["name"].lower()))
+        host_id = node_host.get(n.node_id)
+        cores, mem = capacity.get(host_id, (None, None)) if host_id else (None, None)
+        measured = [r for r in tasks if r["measured"]]
+        cards.append({
+            "node_id": n.node_id, "hostname": n.hostname or n.node_id[:12], "role": n.role,
+            "leader": n.leader, "status": n.status, "availability": n.availability, "ok": node_ok(n),
+            "host_id": host_id, "host_name": host_names.get(host_id) if host_id else None,
+            "agent": bool(running_on.get(n.node_id)) or host_id is not None,
+            "tasks": tasks, "task_count": sum(r["tasks"] for r in tasks),
+            "cores_used": sum(r["cores"] for r in measured) if measured else None,
+            "mem_used": sum(r["mem_bytes"] for r in measured) if measured else None,
+            "cores": cores, "mem": mem,
+        })
+    return sorted(cards, key=lambda c: (c["ok"], c["role"] != "manager", c["hostname"].lower()))

plugins/docker/swarm_view.py

@@ -11,12 +11,28 @@ For every swarm service we list the real replicas we have detail for, then add
 "ghost" replicas for the remaining placement count on each node (agent-less
 nodes). Non-swarm containers pass through grouped by host, unchanged.
 
+A third source, DockerSwarmService.tasks_json (agent ≥ 1.8.0), is Swarm's own
+record of each service's newest tasks. It decides which task containers are
+*superseded* — replaced by Swarm, so neither a replica nor a crash — and gives
+each service its failures across task retries and the error of its last
+failed task. Swarm restarts a task by creating a new one (a new container), so
+failures can only be counted per service, never per container.
+
 Kept model-free (no ORM imports) so it's unit-testable and safe to import in
 tests without the plugin-loader "table already defined" gotcha.
 """
 from __future__ import annotations
 
 import json
+from datetime import datetime, timedelta, timezone
+
+# Task states that mean the task ended in error: `failed` (its container exited
+# non-zero or never started) and `rejected` (Swarm could not start it at all,
+# e.g. no such image, a bad mount).
+FAILED_TASK_STATES = ("failed", "rejected")
+# Desired states of a task Swarm still means to run. Anything else (shutdown,
+# remove, complete, orphaned) is a task Swarm has finished with.
+_CURRENT_DESIRED = ("running", "ready", "")
 
 
 def _service_state(running: int, desired: int) -> str:
@@ -36,30 +52,222 @@ def _placement(service) -> list[dict]:
     return [p for p in data if isinstance(p, dict)]
 
 
-def build_swarm_services(containers, services, node_hostname: dict, host_name: dict) -> dict:
-    """Return {"services": [...], "standalone": [...]}.
+def parse_tasks(service) -> list[dict]:
+    """A service row's task records, newest first; [] when its agent sent none."""
+    raw = getattr(service, "tasks_json", None) or "[]"
+    try:
+        data = json.loads(raw)
+    except (ValueError, TypeError):
+        return []
+    return [t for t in data if isinstance(t, dict)]
 
-    services[i] = {name, mode, desired, running, state, replicas:[...]}
-      real replica  = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
-                       health, restart_count}
-      ghost replica = {ghost:True, host, count}   # placement with no local row
-    standalone[i] = {host_id, host, containers:[...]}  # non-swarm, by host
+
+def task_time(task: dict) -> datetime | None:
+    """A task's status timestamp. Docker writes nanoseconds; Python ≥ 3.11 keeps
+    the first six digits."""
+    raw = task.get("at") or ""
+    try:
+        at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return at if at.tzinfo else at.replace(tzinfo=timezone.utc)
+
+
+def task_failures(tasks: list[dict], since: datetime) -> int:
+    """How many tasks ended in error at or after `since`."""
+    n = 0
+    for t in tasks:
+        at = task_time(t)
+        if t.get("state") in FAILED_TASK_STATES and at is not None and at >= since:
+            n += 1
+    return n
+
+
+def last_task_error(tasks: list[dict]) -> dict | None:
+    """Why the service's newest troubled task is in trouble, from Swarm itself.
+
+    The newest task that failed or was rejected, or that Swarm attached an error
+    to while still trying (a pending task with "no suitable node", say). None
+    when no retained task has a problem.
     """
+    for t in tasks:  # newest first
+        if t.get("state") in FAILED_TASK_STATES or t.get("error"):
+            return {"state": t.get("state"), "error": t.get("error") or None,
+                    "message": t.get("message") or None, "exit_code": t.get("exit_code"),
+                    "node_id": t.get("node_id") or None, "at": t.get("at") or None,
+                    "task_id": t.get("task_id") or None, "slot": t.get("slot")}
+    return None
+
+
+def task_summary(tasks: list[dict], now: datetime, crash_loop_deaths: int,
+                 node_hostname: dict) -> dict:
+    """What a service's task records say: tasks_known, failures_1h,
+    failures_24h (None without records), crash_looping, and last_error with
+    the node's hostname as `node`."""
+    known = bool(tasks)
+    failures_1h = task_failures(tasks, now - timedelta(hours=1)) if known else None
+    last_error = last_task_error(tasks)
+    if last_error is not None:
+        nid = last_error["node_id"]
+        last_error["node"] = node_hostname.get(nid) or (nid[:12] if nid else None)
+        at = task_time(last_error)
+        last_error["age_s"] = max(0, int((now - at).total_seconds())) if at else None
+    return {
+        "tasks_known": known,
+        "failures_1h": failures_1h,
+        "failures_24h": task_failures(tasks, now - timedelta(hours=24)) if known else None,
+        "crash_looping": bool(known and failures_1h >= crash_loop_deaths),
+        "last_error": last_error,
+    }
+
+
+def _oldest_failure(tasks: list[dict], since: datetime) -> datetime | None:
+    times = [at for t in tasks if t.get("state") in FAILED_TASK_STATES
+             and (at := task_time(t)) is not None and at >= since]
+    return min(times, default=None)
+
+
+def trouble_since(tasks: list[dict], state: str, crash_looping: bool,
+                  now: datetime) -> datetime | None:
+    """When a service's current trouble began, as far as its retained tasks
+    tell; None when it has none, or the tasks cannot say.
+
+    A crash loop dates from its first failure in the last hour, and a healthy
+    service that lost tasks from its first failure in the last 24h. A service
+    short of replicas dates from its troubled slots (replicated: the slot;
+    global: the node): in each slot whose newest task is not running, the
+    oldest of its newest troubled tasks (failed, rejected, or carrying an
+    error), walking back until a running task interrupts them. Swarm keeps few
+    old tasks, so a long outage reads as the oldest one retained: a lower
+    bound.
+    """
+    if crash_looping:
+        return _oldest_failure(tasks, now - timedelta(hours=1))
+    if state == "healthy":
+        return _oldest_failure(tasks, now - timedelta(hours=24))
+    by_slot: dict = {}
+    for t in tasks:  # newest first, so each slot's list is too
+        by_slot.setdefault(t.get("slot") or t.get("node_id"), []).append(t)
+    began = []
+    for slot_tasks in by_slot.values():
+        oldest = None
+        for t in slot_tasks:
+            if t.get("state") == "running":
+                break
+            if t.get("state") in FAILED_TASK_STATES or t.get("error"):
+                oldest = task_time(t) or oldest
+        if oldest is not None:
+            began.append(oldest)
+    return min(began, default=None)
+
+
+def _slot_key(c) -> str:
+    """A task container's slot within its service: "<service>.<slot>.<task>" for
+    replicated services, "<service>.<node id>.<task>" for global ones."""
+    svc, name = c.service_name, c.name or ""
+    rest = name[len(svc) + 1:] if name.startswith(svc + ".") else name
+    return rest.split(".", 1)[0]
+
+
+def superseded_containers(swarm_cs, tasks_by_id: dict[str, dict]) -> set[tuple]:
+    """(host_id, name) of task containers Swarm has replaced.
+
+    With the task on record, its desired state decides: Swarm sets a replaced
+    or failed task's desired state to shutdown. Without it (an older agent, or a
+    task older than the retained history), a container that is not running is
+    superseded when another task in the same slot is running or started later.
+    """
+    out: set[tuple] = set()
+    by_slot: dict[tuple, list] = {}
+    for c in swarm_cs:
+        by_slot.setdefault((c.service_name, _slot_key(c)), []).append(c)
+    for c in swarm_cs:
+        task = tasks_by_id.get(getattr(c, "task_id", None) or "")
+        if task is not None:
+            if task.get("desired_state") not in _CURRENT_DESIRED:
+                out.add((c.host_id, c.name))
+            continue
+        if (getattr(c, "status", "") or "").lower() == "running":
+            continue
+        mine = getattr(c, "started_at", None)
+        for other in by_slot[(c.service_name, _slot_key(c))]:
+            if other is c:
+                continue
+            theirs = getattr(other, "started_at", None)
+            if ((getattr(other, "status", "") or "").lower() == "running"
+                    or (theirs is not None and (mine is None or theirs > mine))):
+                out.add((c.host_id, c.name))
+                break
+    return out
+
+
+def _newest_per_name(services) -> dict[str, object]:
+    """Every manager reports the same cluster-global service set; keep the most
+    recently updated row per name, so a manager on an older agent (no tasks)
+    doesn't shadow one that reports them."""
+    by_name: dict[str, object] = {}
+    for s in services:
+        name = getattr(s, "service_name", None)
+        if not name:
+            continue
+        kept = by_name.get(name)
+        if kept is None:
+            by_name[name] = s
+            continue
+        a, b = getattr(s, "updated_at", None), getattr(kept, "updated_at", None)
+        if a is not None and (b is None or a > b):
+            by_name[name] = s
+    return by_name
+
+
+def replaced_tasks(containers, services) -> set[tuple]:
+    """(host_id, name) of the task containers Swarm has replaced, from the
+    container rows and the managers' service rows. What the container views use
+    to leave history out of their problem lists."""
+    swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
+    tasks_by_id = {t["task_id"]: t for s in _newest_per_name(services).values()
+                   for t in parse_tasks(s) if t.get("task_id")}
+    return superseded_containers(swarm_cs, tasks_by_id)
+
+
+def build_swarm_services(containers, services, node_hostname: dict, host_name: dict,
+                         *, now: datetime | None = None, crash_loop_deaths: int = 3) -> dict:
+    """Return {"services": [...], "standalone": [...], "superseded": {(host_id, name)}}.
+
+    services[i] = {name, mode, image, desired, running, state, replicas:[...],
+                   previous_tasks, tasks_known, failures_1h, failures_24h,
+                   crash_looping, last_error}
+      real replica  = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
+                       mem_bytes, started_at, image, health, restart_count}
+                      # current tasks only
+      ghost replica = {ghost:True, host, count}   # placement with no local row
+      previous_tasks = task containers Swarm replaced, left out of replicas
+      failures_*    = tasks that failed or were rejected in the window (None
+                      without task data); crash_looping = failures_1h ≥ the
+                      Settings → Thresholds crash-loop count
+      last_error    = last_task_error(), plus `node` (the node's hostname)
+      trouble_since = trouble_since(): when the current trouble began, or None
+    standalone[i] = {host_id, host, containers:[...]}  # non-swarm, by host
+    superseded    = the task containers Swarm replaced, so callers can skip them
+    """
+    now = now or datetime.now(timezone.utc)
     containers = list(containers)
     swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
     standalone_cs = [c for c in containers if not getattr(c, "service_name", None)]
 
-    # Dedup service rows by name (every manager reports the same cluster-global
-    # set); keep the first — placement is identical across managers.
-    svc_by_name: dict[str, object] = {}
-    for s in services:
-        name = getattr(s, "service_name", None)
-        if name and name not in svc_by_name:
-            svc_by_name[name] = s
+    svc_by_name = _newest_per_name(services)
+    tasks_by_svc = {name: parse_tasks(s) for name, s in svc_by_name.items()}
+    tasks_by_id = {t["task_id"]: t for ts in tasks_by_svc.values() for t in ts
+                   if t.get("task_id")}
+    superseded = superseded_containers(swarm_cs, tasks_by_id)
 
-    # Real containers grouped by service name.
+    # Current task containers grouped by service name; replaced ones are counted.
     cs_by_service: dict[str, list] = {}
+    previous: dict[str, int] = {}
     for c in swarm_cs:
+        if (c.host_id, c.name) in superseded:
+            previous[c.service_name] = previous.get(c.service_name, 0) + 1
+            continue
         cs_by_service.setdefault(c.service_name, []).append(c)
 
     def _host_of(c):
@@ -82,6 +290,9 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
                 "status": getattr(c, "status", "unknown"),
                 "cpu_pct": getattr(c, "cpu_pct", None),
                 "mem_pct": getattr(c, "mem_pct", None),
+                "mem_bytes": getattr(c, "mem_usage_bytes", None),
+                "started_at": getattr(c, "started_at", None),
+                "image": getattr(c, "image", None) or None,
                 "health": getattr(c, "health", None),
                 "restart_count": getattr(c, "restart_count", 0) or 0,
             })
@@ -117,13 +328,20 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
         if desired is None:
             desired = len(replicas)
 
+        tasks = tasks_by_svc.get(name, [])
+        state = _service_state(int(running or 0), int(desired or 0))
+        summary = task_summary(tasks, now, crash_loop_deaths, node_hostname)
         out_services.append({
             "name": name,
             "mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
+            "image": getattr(svc, "image", None) if svc is not None else None,
             "running": running,
             "desired": desired,
-            "state": _service_state(int(running or 0), int(desired or 0)),
+            "state": state,
             "replicas": replicas,
+            "previous_tasks": previous.get(name, 0),
+            **summary,
+            "trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
         })
 
     # Non-swarm containers grouped by host, running-first order preserved.
@@ -137,4 +355,34 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
         g["containers"].append(c)
     standalone = sorted(groups.values(), key=lambda g: g["host"].lower())
 
-    return {"services": out_services, "standalone": standalone}
+    return {"services": out_services, "standalone": standalone, "superseded": superseded}
+
+
+def service_trouble(s: dict) -> int | None:
+    """How bad a built service is, for sorting and for Needs attention: 0 a
+    crash loop, 1 no replica running, 2 short of replicas, 3 healthy but with
+    failed tasks in the last 24h; None when nothing is wrong."""
+    if s.get("crash_looping"):
+        return 0
+    if s.get("state") == "down":
+        return 1
+    if s.get("state") == "degraded":
+        return 2
+    if s.get("failures_24h"):
+        return 3
+    return None
+
+
+def services_problems_first(services: list[dict]) -> list[dict]:
+    """Services in trouble first, worst first, then the rest by name."""
+    return sorted(services, key=lambda s: (
+        4 if service_trouble(s) is None else service_trouble(s), s["name"].lower()))
+
+
+def failing_services(services: list[dict]) -> list[dict]:
+    """The services that belong in Needs attention: a crash loop, or short of
+    their desired replicas. A healthy service whose failed tasks Swarm has
+    already replaced is history, shown on the service, not a problem now."""
+    # A crash loop ranks 0, so test the rank against None, not for truth.
+    return [s for s in services_problems_first(services)
+            if (t := service_trouble(s)) is not None and t <= 2]

plugins/docker/templates/docker/_container_logs_lines.html

@@ -0,0 +1,24 @@
+{# docker/_container_logs_lines.html — log-line fragment, HTMX-polled (m79).
+   {{ l.line }} is auto-escaped by Jinja, so log content can't inject markup.
+   A service's lines (#5302) carry the task each came from. #}
+{% if lines %}
+{% for l in lines %}
+<div style="display:flex;gap:0.6rem;white-space:pre-wrap;word-break:break-word;">
+  <span style="color:var(--text-dim);flex-shrink:0;" title="{{ l.ts }}">{{ l.ts.strftime("%m-%d %H:%M:%S") }}</span>
+  {% if l.task %}<span class="tone-muted" style="flex-shrink:0;white-space:nowrap;" title="Slot and task id">{{ l.task }}</span>{% endif %}
+  <span style="flex-shrink:0;width:3.2rem;color:{% if l.stream == 'stderr' %}var(--red){% else %}var(--text-muted){% endif %};">{{ l.stream }}</span>
+  <span style="flex:1;min-width:0;{% if l.stream == 'stderr' %}color:var(--text);{% endif %}">{{ l.line }}</span>
+</div>
+{% endfor %}
+{% else %}
+<div style="color:var(--text-muted);padding:1rem 0.25rem;">
+  {% if filtered %}
+  No lines match the current filter.
+  {% else %}
+  No logs collected yet for <code>{{ name }}</code>. Lines appear within a few
+  seconds of {{ "one of its tasks" if service else "the container" }} writing to stdout/stderr — unless it's on the log
+  exclude list or log collection is turned off in
+  <a href="/settings/thresholds/">Settings → Thresholds &amp; Retention</a>.
+  {% endif %}
+</div>
+{% endif %}

plugins/docker/templates/docker/_service_history.html

@@ -0,0 +1,22 @@
+{# docker/_service_history.html — a service's CPU and memory summed across its
+   replicas over the selected range (#5302): the latest and the peak, over a
+   line that spans its half. #}
+{% from "docker/_usage.html" import cores %}
+{% if have_data %}
+<div class="history-pair">
+  <div>
+    <div class="section-title">CPU</div>
+    <div style="font-variant-numeric:tabular-nums;">{{ cores(cpu_now) }} <span class="tone-dim">now · peak {{ cores(cpu_peak) }}</span></div>
+    <div class="spark">{{ spark_cpu|safe }}</div>
+  </div>
+  <div>
+    <div class="section-title">Memory</div>
+    <div style="font-variant-numeric:tabular-nums;">{{ mem_now|bytes }} <span class="tone-dim">now · peak {{ mem_peak|bytes }}</span></div>
+    <div class="spark">{{ spark_mem|safe }}</div>
+  </div>
+</div>
+{% else %}
+<p class="tone-muted" style="margin:0;font-size:0.85rem;">
+  Not enough samples in the last {{ range_key }} yet. History fills in as the agents report its replicas.
+</p>
+{% endif %}

plugins/docker/templates/docker/_swarm_failures.html

@@ -0,0 +1,25 @@
+{#- One line on a swarm service's task failures, from Swarm's own task records
+    (agent ≥ 1.8.0): a crash loop or the failures of the last 24h, and how the
+    newest troubled task ended, in Swarm's words. Expects `s` with state,
+    crash_looping, failures_1h, failures_24h and last_error; `style` is
+    optional extra CSS for the wrapper. Renders nothing for a healthy service
+    with no recent failures. -#}
+{%- set e = s.last_error -%}
+{%- if s.crash_looping or s.failures_24h or (e and s.state != 'healthy') %}
+<div style="display:flex;flex-wrap:wrap;align-items:baseline;gap:0.2rem 0.6rem;font-size:0.78rem;color:var(--text-muted);{{ style or '' }}">
+  {% if s.crash_looping %}
+  <span style="color:var(--red);font-weight:500;">Crash-looping · {{ s.failures_1h }} tasks failed in the last hour</span>
+  {% elif s.failures_24h %}
+  <span style="color:var(--orange);">{{ s.failures_24h }} task{{ 's' if s.failures_24h != 1 }} failed in the last 24h</span>
+  {% endif %}
+  {% if e %}
+  <span title="How Swarm recorded the end of this service's newest failed task">
+    Last task {{ e.state }}:
+    <span style="color:var(--text);font-family:ui-monospace,monospace;overflow-wrap:anywhere;">{{ e.error or e.message or e.state }}</span>
+    {%- if e.exit_code %} · exit {{ e.exit_code }}{% endif %}
+    {%- if e.node %} · {{ e.node }}{% endif %}
+    {%- if e.age_s is not none %} · {{ e.age_s|duration }} ago{% endif %}
+  </span>
+  {% endif %}
+</div>
+{%- endif %}

plugins/docker/templates/docker/_usage.html

@@ -0,0 +1,26 @@
+{#- A container's CPU and memory as service.usage() describes them (#5209).
+
+    Docker counts CPU per core (100% = one core busy), so CPU reads as cores,
+    toned by its share of the host's CPUs when the host's core count is known.
+    Memory reads in bytes, against a limit only when one is actually set;
+    mem_pct is then a share of that limit, otherwise of the host's memory. -#}
+
+{% macro cores(n) -%}
+{{ "%.2f"|format(n) if n < 1 else "%.1f"|format(n) }} {{ "core" if n == 1 else "cores" }}
+{%- endmacro %}
+
+{% macro cpu(u) -%}
+{%- if u.cores is none -%}<span class="tone-dim">—</span>
+{%- else -%}
+<span class="tone-{{ status_class(u.cpu_share, 'cpu') }}"
+      title="Docker counts CPU per core: {{ '%.0f'|format(u.cores * 100) }}% of one core{% if u.cpus %}, {{ '%.1f'|format(u.cpu_share) }}% of this host's {{ u.cpus }} core{{ 's' if u.cpus != 1 }}{% endif %}">{{ cores(u.cores) }}</span>
+{%- endif -%}
+{%- endmacro %}
+
+{% macro mem(u) -%}
+{%- if u.mem_bytes is none -%}<span class="tone-dim">—</span>
+{%- else -%}
+<span class="tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of else 'dim' }}"
+      title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% else %}No memory limit set{% endif %}">{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %}</span>
+{%- endif -%}
+{%- endmacro %}

plugins/docker/templates/docker/container_detail.html

@@ -1,13 +1,16 @@
-{# docker/container_detail.html — full detail page for one container #}
+{# docker/container_detail.html — full detail page for one container. A swarm
+   task is titled by its service and slot, with the raw container name under
+   it; CPU reads in cores and memory in bytes (see _usage.html). #}
 {% extends "base.html" %}
 {% from "_macros.html" import crumbs %}
-{% block title %}{{ name }} — Docker — Steward{% endblock %}
-{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
+{% from "docker/_usage.html" import cpu, mem %}
+{% block title %}{{ label or name }} — Docker — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (label or name, "")]) }}{% endblock %}
 {% block content %}
 
 {% if container is none %}
 <div class="card" style="text-align:center;padding:3rem;">
-  <div style="font-weight:600;margin-bottom:0.4rem;">Container not found</div>
+  <div style="font-weight:500;margin-bottom:0.4rem;">Container not found</div>
   <div style="color:var(--text-muted);font-size:0.9rem;">
     No container named <code>{{ name }}</code> is currently reported for this host.
     It may have been removed, or the host agent hasn't reported recently.
@@ -18,14 +21,18 @@
 {# ── Header ──────────────────────────────────────────────────────────────── #}
 <div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
   <span class="dot {% if container.status == 'running' %}dot-up{% elif container.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-  <h1 class="page-title" style="margin-bottom:0;">{{ container.name }}</h1>
+  <h1 class="page-title" style="margin-bottom:0;">{{ label }}</h1>
   {% if container.health == 'healthy' %}<span style="font-size:0.72rem;color:var(--green);border:1px solid var(--green);border-radius:3px;padding:0.05rem 0.4rem;">healthy</span>
   {% elif container.health == 'unhealthy' %}<span style="font-size:0.72rem;color:var(--red);border:1px solid var(--red);border-radius:3px;padding:0.05rem 0.4rem;">unhealthy</span>
   {% elif container.health == 'starting' %}<span style="font-size:0.72rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0.05rem 0.4rem;">starting</span>{% endif %}
 </div>
+{% if label != container.name %}
+<div style="font-size:0.75rem;color:var(--text-dim);font-family:ui-monospace,monospace;margin-bottom:0.25rem;overflow-wrap:anywhere;">{{ container.name }}</div>
+{% endif %}
 <div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
   {{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %}
   {% if host %} · on <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %}
+  · <a href="/plugins/docker/container/{{ host_id }}/{{ name }}/logs">View logs</a>
 </div>
 
 {# ── Facts grid ──────────────────────────────────────────────────────────── #}
@@ -37,16 +44,27 @@
 {% endmacro %}
 
 <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
-  {{ fact("CPU", "%.1f%%" | format(container.cpu_pct) if container.cpu_pct is not none else "—") }}
-  {{ fact("Memory", "%.1f%%" | format(container.mem_pct) if container.mem_pct is not none else "—") }}
+  {{ fact("CPU", cpu(usage) if container.status == 'running' else "—") }}
+  {{ fact("Memory", mem(usage) if container.status == 'running' else "—") }}
+  {#- Swarm never restarts a task's container: it replaces the task with a new
+      one, so restart_count is always 0 there and the service's task failures
+      (Swarm page) are the count that means something. #}
+  {% if not container.service_name %}
   {{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }}
+  {% endif %}
   {{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—",
-          "var(--red)" if (container.exit_code is not none and container.exit_code != 0) else "") }}
-  {{ fact("Net in", net_rx) }}
-  {{ fact("Net out", net_tx) }}
-  {{ fact("Block read", blk_read) }}
-  {{ fact("Block write", blk_write) }}
+          "var(--red)" if exit_failed else "") }}
+  {{ fact("Net in since start", net_rx) }}
+  {{ fact("Net out since start", net_tx) }}
+  {{ fact("Disk read since start", blk_read) }}
+  {{ fact("Disk written since start", blk_write) }}
 </div>
+{% if container.service_name %}
+<div style="font-size:0.78rem;color:var(--text-muted);margin:-0.75rem 0 1.5rem;">
+  Swarm replaces a failed task with a new container rather than restarting it; the
+  service's failed tasks are on the <a href="/plugins/docker/swarm">Swarm page</a>.
+</div>
+{% endif %}
 
 {# ── Image / placement ───────────────────────────────────────────────────── #}
 <div class="card" style="margin-bottom:1.5rem;">
@@ -60,7 +78,9 @@
     <span style="color:var(--text-muted);">Swarm service</span><span>{{ container.service_name }}</span>
     {% endif %}
     {% if container.node_id %}
-    <span style="color:var(--text-muted);">Node</span><span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>
+    <span style="color:var(--text-muted);">Node</span>
+    {% if node_name %}<span title="node id {{ container.node_id }}">{{ node_name }}</span>
+    {% else %}<span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>{% endif %}
     {% endif %}
     {% if ports %}
     <span style="color:var(--text-muted);">Ports</span>

plugins/docker/templates/docker/container_logs.html

@@ -0,0 +1,51 @@
+{# docker/container_logs.html — per-container log viewer (m79) #}
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs %}
+{% block title %}Logs — {{ name }} — Docker — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([
+  ("Docker", "/plugins/docker/"),
+  (name, "/plugins/docker/container/" ~ host_id ~ "/" ~ name),
+  ("Logs", "")]) }}{% endblock %}
+{% block content %}
+
+<div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
+  <h1 class="page-title" style="margin-bottom:0;">{{ name }}</h1>
+  <span style="font-size:0.9rem;color:var(--text-muted);">logs</span>
+</div>
+<div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1rem;">
+  Recent lines (newest first) collected by the host agent{% if host %} on
+  <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %} —
+  updated every few seconds.
+  <a href="/plugins/docker/container/{{ host_id }}/{{ name }}">← back to container</a>
+</div>
+
+{# ── Controls: stream filter + text search (drive the fragment via HTMX) ───── #}
+<form id="log-controls" onsubmit="return false;"
+      style="display:flex;gap:0.6rem;align-items:center;flex-wrap:wrap;margin-bottom:0.6rem;">
+  <label style="font-size:0.8rem;color:var(--text-muted);display:flex;align-items:center;gap:0.35rem;">
+    Stream
+    <select name="stream" style="padding:0.25rem 0.4rem;">
+      <option value="all">all</option>
+      <option value="stdout">stdout</option>
+      <option value="stderr">stderr</option>
+    </select>
+  </label>
+  <input type="search" name="q" placeholder="Filter lines…" autocomplete="off"
+         aria-label="Filter log lines"
+         style="flex:1;min-width:180px;padding:0.3rem 0.5rem;">
+</form>
+
+<div class="card-flush">
+  <div id="log-lines"
+       hx-get="/plugins/docker/container/{{ host_id }}/{{ name }}/logs/lines"
+       hx-trigger="load, every 5s, change from:#log-controls, keyup changed delay:400ms from:#log-controls"
+       hx-include="#log-controls"
+       hx-swap="innerHTML"
+       role="log" aria-live="polite" tabindex="0"
+       style="max-height:70vh;overflow:auto;padding:0.5rem 0.75rem;
+              font-family:ui-monospace,monospace;font-size:0.8rem;line-height:1.5;">
+    <div style="color:var(--text-muted);">Loading…</div>
+  </div>
+</div>
+
+{% endblock %}

plugins/docker/templates/docker/disk.html

@@ -7,20 +7,22 @@
 
 <h1 class="page-title" style="margin-bottom:0.4rem;">Image &amp; disk usage</h1>
 <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
-  Reclaimable = space held by images no container references. Cleanup actions
-  (prune) arrive in a later release — these are read-only figures for now.
+  What Docker is keeping on each host's disk, from <code>docker system df</code>.
+  Admins can clean up per host below; each action runs an audited Ansible playbook on that host.
 </p>
 
 {% if host_groups %}
 {% for g in host_groups %}
 <div style="margin-bottom:2rem;">
-  <div style="font-weight:600;font-size:0.95rem;margin-bottom:0.75rem;">{{ g.host_name }}</div>
+  <div style="font-weight:500;font-size:0.95rem;margin-bottom:0.75rem;">{{ g.host_name }}
+    {% if g.as_of %}<span class="field-hint" style="font-weight:normal;">· as of {{ g.as_of|ago }}</span>{% endif %}</div>
 
   {# ── Summary stats ────────────────────────────────────────────────────── #}
   <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1rem;">
     <div class="card" style="margin-bottom:0;">
       <div class="section-title" style="margin-bottom:0.3rem;">Reclaimable</div>
       <span class="stat-val" style="color:{% if g.summary.images_reclaimable != '0 B' %}var(--orange){% else %}var(--green){% endif %};">{{ g.summary.images_reclaimable }}</span>
+      <div style="font-size:0.74rem;color:var(--text-muted);">held by images no container uses</div>
     </div>
     <div class="card" style="margin-bottom:0;">
       <div class="section-title" style="margin-bottom:0.3rem;">Images</div>
@@ -30,39 +32,81 @@
     <div class="card" style="margin-bottom:0;">
       <div class="section-title" style="margin-bottom:0.3rem;">Stopped</div>
       <span class="stat-val" style="{% if g.stopped %}color:var(--text-muted){% endif %};">{{ g.stopped }}</span>
+      <div style="font-size:0.74rem;color:var(--text-muted);">containers not running</div>
     </div>
     <div class="card" style="margin-bottom:0;">
-      <div class="section-title" style="margin-bottom:0.3rem;">Writable layers</div>
+      <div class="section-title" style="margin-bottom:0.3rem;">Container files</div>
       <span class="stat-val">{{ g.summary.containers_size }}</span>
-      <div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.containers_count }} containers</div>
+      <div style="font-size:0.74rem;color:var(--text-muted);">written by {{ g.summary.containers_count }} containers on top of their images</div>
     </div>
     <div class="card" style="margin-bottom:0;">
       <div class="section-title" style="margin-bottom:0.3rem;">Volumes</div>
       <span class="stat-val">{{ g.summary.volumes_size }}</span>
-      <div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.volumes_count }} volumes</div>
+      <div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.volumes_count }} volumes; never pruned here</div>
     </div>
     <div class="card" style="margin-bottom:0;">
       <div class="section-title" style="margin-bottom:0.3rem;">Build cache</div>
       <span class="stat-val">{{ g.summary.build_cache_size }}</span>
+      <div style="font-size:0.74rem;color:var(--text-muted);">left by <code>docker build</code>; safe to clear</div>
     </div>
   </div>
 
+  {# ── Cleanup actions (admin only; audited Ansible prune run) ───────────── #}
+  {% if session.user_role == 'admin' %}
+  <div style="margin-bottom:1rem;">
+    {% if ansible_available and g.has_target %}
+    <div style="display:flex;flex-wrap:wrap;gap:0.5rem;align-items:center;">
+      <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
+            data-confirm="Remove all STOPPED containers on {{ g.host_name }}? This cannot be undone.">{{ csrf_field() }}
+        <input type="hidden" name="target" value="containers">
+        <button type="submit" class="btn btn-sm">Prune stopped containers</button>
+      </form>
+      <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
+            data-confirm="Remove ALL unused images on {{ g.host_name }} (docker image prune -a)? Any image not used by a container is deleted.">{{ csrf_field() }}
+        <input type="hidden" name="target" value="images">
+        <button type="submit" class="btn btn-sm">Prune unused images</button>
+      </form>
+      <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
+            data-confirm="Run a full system prune on {{ g.host_name }}? Removes stopped containers, unused networks, untagged images nothing uses, and the build cache.">{{ csrf_field() }}
+        <input type="hidden" name="target" value="system">
+        <button type="submit" class="btn btn-sm btn-danger">System prune…</button>
+      </form>
+    </div>
+    <div style="font-size:0.72rem;color:var(--text-muted);margin-top:0.4rem;">
+      Reclaimed space appears on the next agent sample.
+    </div>
+    {% elif not ansible_available %}
+    <div style="font-size:0.74rem;color:var(--text-muted);">Cleanup needs the Ansible runner (currently unavailable).</div>
+    {% else %}
+    <div style="font-size:0.74rem;color:var(--text-muted);">
+      Link an <a href="/hosts/{{ g.host_id }}">Ansible target</a> to this host to enable prune actions.
+    </div>
+    {% endif %}
+  </div>
+  {% endif %}
+
   {# ── Per-image table ──────────────────────────────────────────────────── #}
+  <div class="field-hint" style="margin-bottom:0.4rem;">
+    {% if g.images_capped %}The {{ g.images|length }} largest of {{ g.summary.images_total }} images, largest first.
+    {% else %}Every image, largest first.{% endif %}
+  </div>
   <div class="card-flush">
     <table class="table">
       <thead>
-        <tr><th>Image</th><th>Size</th><th>Shared</th><th>Containers</th></tr>
+        <tr><th>Image</th><th>Size</th>
+          <th title="Layers this image has in common with other images: stored once, so removing this image frees its size less this">Shared with other images</th>
+          <th>Used by</th></tr>
       </thead>
       <tbody>
         {% for im in g.images %}
         <tr>
           <td style="font-size:0.84rem;font-family:ui-monospace,monospace;max-width:340px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
-            {{ im.repo_tag }}
-            {% if im.reclaimable %}<span title="not referenced by any container" style="font-size:0.68rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0 0.3rem;margin-left:0.4rem;">reclaimable</span>{% endif %}
+            {% if im.repo_tag %}{{ im.repo_tag }}{% else %}<span style="color:var(--text-muted);">untagged</span> {{ im.image_id }}{% endif %}
+            {% if im.reclaimable %}<span title="No container uses this image" style="font-size:0.68rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0 0.3rem;margin-left:0.4rem;">reclaimable</span>{% endif %}
           </td>
           <td style="font-family:ui-monospace,monospace;font-size:0.86rem;">{{ im.size }}</td>
           <td style="font-family:ui-monospace,monospace;font-size:0.86rem;color:var(--text-muted);">{{ im.shared }}</td>
-          <td style="font-size:0.86rem;color:{% if im.containers %}var(--text){% else %}var(--text-muted){% endif %};">{{ im.containers }}</td>
+          <td style="font-size:0.86rem;color:{% if im.containers %}var(--text){% else %}var(--text-muted){% endif %};">{% if im.containers %}{{ im.containers }} container{{ "s" if im.containers != 1 }}{% else %}nothing{% endif %}</td>
         </tr>
         {% else %}
         <tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No images reported.</td></tr>

plugins/docker/templates/docker/host_panel.html

@@ -1,22 +1,76 @@
-{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub #}
+{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub:
+   problems first with what is wrong and since when, then the busiest running
+   containers; the rest of the running and the stopped fold away so the card
+   stays short (#5276). Tasks Swarm has replaced are left out (counted in the
+   header). #}
+{% from "_macros.html" import status_badge, status_dot %}
+{% from "docker/_usage.html" import cpu, mem %}
+{% macro container_table(rows) %}
+<table class="table">
+  <colgroup><col><col style="width:7rem"><col style="width:9rem"></colgroup>
+  <thead>
+    <tr>
+      <th>Container</th>
+      <th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
+      <th title="Used, and the limit when one is set">Memory</th>
+    </tr>
+  </thead>
+  <tbody>
+      {% for r in rows %}
+      {% set c = r.c %}
+      <tr>
+        <td>
+          <div style="display:flex;align-items:center;gap:0.5rem;min-width:0;">
+            {{ status_dot('crit' if r.problems else ('up' if c.status == 'running' else 'inactive'), c.status|capitalize) }}
+            <div style="min-width:0;">
+              <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }} · {{ c.image }}" style="font-weight:500;color:inherit;text-decoration:none;overflow-wrap:anywhere;">{{ r.label }}</a>
+              {% if r.problems %}
+              {% set worst = r.problems[0] %}
+              <div style="font-size:0.75rem;margin-top:0.1rem;">
+                {{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
+                <span style="color:var(--text-muted);">
+                  {%- for p in r.problems %} {{ says[p] }}{{ " and" if not loop.last }}{% endfor %}
+                  {%- if c.exit_code %} · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}{% endif %}
+                  {%- if r.since %} · since {{ r.since|ago }}{% endif -%}
+                </span>
+              </div>
+              {% else %}
+              <div style="font-size:0.72rem;color:var(--text-muted);">
+                {{ c.status }}{% if r.uptime %} · up {{ r.uptime }}{% endif %}
+                {%- if c.status != 'running' and c.exit_code %} · exit {{ c.exit_code }}{% endif %}
+              </div>
+              {% endif %}
+            </div>
+          </div>
+        </td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+      </tr>
+      {% endfor %}
+  </tbody>
+</table>
+{% endmacro %}
 <div class="card">
-  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;">
+  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;flex-wrap:wrap;">
     <h3 class="section-title" style="margin-bottom:0;">Docker</h3>
     <span style="font-size:0.78rem;color:var(--text-muted);">
+      {% if problems %}<span class="tone-crit">{{ problems }} with problems</span> · {% endif %}
       {{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %}
+      {% if replaced %}<span title="Old swarm tasks Swarm has replaced with new ones: history, not containers that need anything"> · {{ replaced }} replaced task{{ "s" if replaced != 1 }} hidden</span>{% endif %}
       <a href="/plugins/docker/" style="margin-left:0.6rem;color:var(--text-muted);">All →</a>
     </span>
   </div>
-  <div style="display:grid;gap:2px;">
-    {% for c in containers %}
-    <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.85rem;padding:0.2rem 0;">
-      <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-      <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;" title="{{ c.image }}">{{ c.name }}</a>
-      <span style="font-size:0.74rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
-        {% if c.cpu_pct is not none %}{{ "%.1f" | format(c.cpu_pct) }}%{% endif %}
-        {% if c.mem_pct is not none %} · {{ "%.1f" | format(c.mem_pct) }}%{% endif %}
-      </span>
-    </div>
-    {% endfor %}
-  </div>
+  {{ container_table(rows) }}
+  {% if folded %}
+  <details style="margin-top:0.5rem;">
+    <summary style="cursor:pointer;font-size:0.82rem;color:var(--text-muted);">
+      {%- set more_running = folded|selectattr("c.status", "equalto", "running")|list|length -%}
+      {%- set more_stopped = folded|length - more_running -%}
+      {%- if more_running %}+{{ more_running }} more running{% endif -%}
+      {%- if more_running and more_stopped %} · {% endif -%}
+      {%- if more_stopped %}{{ more_stopped }} stopped{% endif -%}
+    </summary>
+    {{ container_table(folded) }}
+  </details>
+  {% endif %}
 </div>

plugins/docker/templates/docker/index.html

@@ -1,24 +1,68 @@
 {% extends "base.html" %}
+{% from "_macros.html" import page_header %}
+{# The Docker page. The filter bar sits outside the polled fragment, so a poll
+   never resets what is being typed; the fragment reads the bar on every poll,
+   and a change re-fetches it and puts the filters in the URL (#5299). #}
 {% block title %}Docker — Steward{% endblock %}
 {% block content %}
-<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1.5rem;gap:1rem;flex-wrap:wrap;">
-  <div style="display:flex;align-items:baseline;gap:1rem;">
-    <h1 class="page-title" style="margin-bottom:0;">Docker</h1>
-    {% if has_swarm %}
-    <a href="/plugins/docker/swarm" style="font-size:0.85rem;color:var(--accent);text-decoration:none;">Swarm →</a>
-    {% endif %}
-    {% if has_disk %}
-    <a href="/plugins/docker/disk" style="font-size:0.85rem;color:var(--accent);text-decoration:none;">Disk →</a>
-    {% endif %}
-  </div>
-  {% include "_time_range.html" %}
-</div>
+{% call page_header("Docker", "What needs attention first, then every service and container.") %}
+  {% if has_swarm %}<a class="btn btn-sm btn-ghost" href="/plugins/docker/swarm">Swarm</a>{% endif %}
+  {% if has_disk %}<a class="btn btn-sm btn-ghost" href="/plugins/docker/disk">Disk</a>{% endif %}
+{% endcall %}
+
+<form id="docker-filters" class="filter-bar" method="get" action="/plugins/docker/" role="search">
+  <input type="search" id="docker-q" name="q" value="{{ f.q }}" class="input"
+         placeholder="Search name, image or stack" aria-label="Search services and containers" autocomplete="off">
+  {% if choices.stacks %}
+  <select name="stack" aria-label="Stack">
+    <option value="">All stacks</option>
+    {% for s in choices.stacks %}<option value="{{ s }}"{% if f.stack == s %} selected{% endif %}>{{ s }}</option>{% endfor %}
+    <option value="{{ no_stack }}"{% if f.stack == no_stack %} selected{% endif %}>No stack</option>
+  </select>
+  {% endif %}
+  {% if choices.hosts|length > 1 %}
+  <select name="host" aria-label="Host or node">
+    <option value="">All hosts</option>
+    {% for h in choices.hosts %}<option value="{{ h }}"{% if f.host == h %} selected{% endif %}>{{ h }}</option>{% endfor %}
+  </select>
+  {% endif %}
+  <select name="state" aria-label="Status">
+    <option value="">Any status</option>
+    {% for key, label in states.items() %}<option value="{{ key }}"{% if f.state == key %} selected{% endif %}>{{ label }}</option>{% endfor %}
+  </select>
+  {% if f.sort != 'status' %}<input type="hidden" name="sort" value="{{ f.sort }}"><input type="hidden" name="dir" value="{{ f.dir }}">{% endif %}
+  <noscript><button type="submit" class="btn btn-sm">Filter</button></noscript>
+</form>
 
 <div id="docker-rows"
      hx-get="/plugins/docker/rows"
-     hx-trigger="load, rangeChange from:body"
-     hx-include="#time-range"
+     hx-include="#docker-filters"
+     hx-trigger="load, every {{ poll_interval }}s, change from:#docker-filters, input changed delay:250ms from:#docker-q, search from:#docker-q"
      hx-swap="innerHTML">
-  <div style="color:var(--text-muted);font-size:0.9rem;padding:2rem;">Loading...</div>
+  <div class="card"><p class="field-hint">Loading…</p></div>
 </div>
+<script>
+(function () {
+  var rows = document.getElementById('docker-rows');
+  var form = document.getElementById('docker-filters');
+  // The filters live in the URL too, so a filtered view can be bookmarked and
+  // Back returns to it.
+  function keepInUrl() {
+    var q = new URLSearchParams(new FormData(form));
+    Array.from(q.keys()).forEach(function (k) { if (!q.get(k)) q.delete(k); });
+    var s = q.toString();
+    history.replaceState(null, '', location.pathname + (s ? '?' + s : '') + location.hash);
+  }
+  form.addEventListener('change', keepInUrl);
+  form.addEventListener('input', keepInUrl);
+  // A link to a section (the dashboard's "failed (24h)" → #failures) lands
+  // before the rows exist; scroll to it once, when they first arrive.
+  if (!location.hash || !rows) return;
+  rows.addEventListener('htmx:afterSwap', function once() {
+    rows.removeEventListener('htmx:afterSwap', once);
+    var target = document.getElementById(location.hash.slice(1));
+    if (target) target.scrollIntoView({ block: 'start' });
+  });
+})();
+</script>
 {% endblock %}

plugins/docker/templates/docker/rows.html

@@ -1,174 +1,233 @@
-{# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #}
+{# docker/rows.html — HTMX fragment for the Docker page (#5298–#5301):
+   a verdict line with only the problem counts that aren't zero; what needs
+   attention (service.needs_attention, the ranking the MCP tools use); hosts
+   whose Docker is not answering; what failed in 24h, each with its logs; then
+   the swarm services and the standalone containers as two tables that share
+   the filter bar's search, filters and sort (overview.py). #}
+{% from "_macros.html" import stat_tile, status_badge, status_dot, sort_th %}
+{% from "docker/_usage.html" import cpu, mem, cores %}
+{% macro th(key, label, cls=none, title=none) -%}
+{{ sort_th(label, sort_url(key), active=f.sort == key, desc=f.dir == 'desc', cls=cls, title=title) }}
+{%- endmacro %}
+{% macro up(row) -%}
+{%- if row.started_at %}<span title="Started {{ row.started_at.strftime('%Y-%m-%d %H:%M UTC') }}">{{ row.started_at|ago|striptags|replace(" ago", "") }}</span>
+{%- else %}<span class="tone-dim">—</span>{% endif -%}
+{%- endmacro %}
 
-{# ── Summary strip ─────────────────────────────────────────────────────────── #}
-<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(140px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
-  <div class="card" style="margin-bottom:0;">
-    <div class="section-title" style="margin-bottom:0.35rem;">Running</div>
-    <span class="stat-val" style="color:var(--green);">{{ running }}</span>
-  </div>
-  <div class="card" style="margin-bottom:0;">
-    <div class="section-title" style="margin-bottom:0.35rem;">Stopped</div>
-    <span class="stat-val" style="{% if stopped %}color:var(--text-muted){% endif %};">{{ stopped }}</span>
-  </div>
-  <div class="card" style="margin-bottom:0;">
-    <div class="section-title" style="margin-bottom:0.35rem;">Total</div>
-    <span class="stat-val">{{ total }}</span>
-  </div>
-  {% if swarm_services %}
-  <div class="card" style="margin-bottom:0;">
-    <div class="section-title" style="margin-bottom:0.35rem;">Services</div>
-    <span class="stat-val">{{ swarm_services | length }}</span>
-  </div>
-  {% endif %}
-  <div class="card" style="margin-bottom:0;">
-    <div class="section-title" style="margin-bottom:0.35rem;">Hosts</div>
-    <span class="stat-val">{{ host_groups | length }}</span>
-  </div>
-</div>
-
-{# ── Swarm services: collapsed per-service, each replica labelled with its host.
-   Ghost replicas (tasks on nodes with no Steward agent) come from the managers'
-   placement data so the picture is cluster-complete. ──────────────────────── #}
-{% if swarm_services %}
-<div class="card-flush" style="margin-bottom:1.5rem;">
-  <div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
-    <span style="font-weight:600;font-size:0.95rem;">Swarm services</span>
-    <a href="/plugins/docker/swarm" style="font-size:0.78rem;color:var(--text-muted);">Topology →</a>
-  </div>
-  <div style="padding:0.5rem 0.75rem;display:grid;gap:0.85rem;">
-    {% for s in swarm_services %}
-    <div>
-      <div style="display:flex;align-items:center;gap:0.55rem;flex-wrap:wrap;margin-bottom:0.35rem;">
-        <span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
-        <span style="font-weight:600;font-size:0.9rem;">{{ s.name }}</span>
-        <span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
-        <span style="font-size:0.78rem;color:{% if s.state == 'healthy' %}var(--green){% elif s.state == 'degraded' %}var(--orange){% else %}var(--red){% endif %};font-variant-numeric:tabular-nums;">
-          {{ s.running }}/{{ s.desired }} running
-        </span>
-      </div>
-      <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(210px,1fr));gap:0.4rem;padding-left:1.1rem;">
-        {% for r in s.replicas %}
-        {% if r.ghost %}
-        <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:5px;padding:0.3rem 0.55rem;" title="Running on a node with no Steward agent — detail unavailable">
-          <span class="dot dot-warn" style="opacity:0.6;"></span>
-          <span style="font-weight:500;">{{ r.host }}</span>
-          <span style="color:var(--text-dim);margin-left:auto;">{{ r.count }} · no agent</span>
-        </div>
-        {% else %}
-        <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px solid var(--border);border-radius:5px;padding:0.3rem 0.55rem;">
-          <span class="dot {% if r.status == 'running' %}dot-up{% elif r.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-          <a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" style="font-weight:500;color:inherit;text-decoration:none;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" title="{{ r.name }}">{{ r.host }}</a>
-          {% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;">{{ "%.0f" | format(r.cpu_pct) }}%</span>{% endif %}
-        </div>
-        {% endif %}
-        {% endfor %}
-      </div>
-    </div>
-    {% endfor %}
-  </div>
+{# ── Verdict ──────────────────────────────────────────────────────────────── #}
+{% set problems = attention|length + failing_services|length %}
+<p class="docker-verdict">
+  {% if problems %}<strong class="tone-crit">{{ problems }} need{{ "s" if problems == 1 }} attention</strong>
+  {% else %}<strong class="tone-ok">Nothing needs attention</strong>{% endif %}
+  {%- if services_total %} · <span class="tone-{{ 'ok' if services_healthy == services_total else 'warn' }}">{{ services_healthy }}/{{ services_total }}</span> services healthy{% endif %}
+  {%- if containers_total %} · {{ running }} container{{ "s" if running != 1 }} running{% if stopped %}, {{ stopped }} stopped{% endif %}{% if services_total %} outside services{% endif %}{% endif %}
+  {%- if docker_hosts %} · {{ docker_hosts }} Docker host{{ "s" if docker_hosts != 1 }}{% endif %}
+</p>
+{% if unhealthy or restarting or failures or stale %}
+<div class="stat-strip">
+  {% if unhealthy %}{{ stat_tile("Unhealthy", unhealthy, state="crit") }}{% endif %}
+  {% if restarting %}{{ stat_tile("Restarting", restarting, state="crit", hint="restarting or crash-looping") }}{% endif %}
+  {% if failures %}{{ stat_tile("Failed (24h)", failures|length, state="warn", hint="listed below, with their logs", href="#failures") }}{% endif %}
+  {% if stale %}{{ stat_tile("Stale", stale, state="stale", hint="no recent report") }}{% endif %}
 </div>
 {% endif %}
 
-{# ── Container tables, one per host ───────────────────────────────────────── #}
-{% if host_groups %}
-{% for g in host_groups %}
-<div class="card-flush" style="margin-bottom:1.5rem;">
-  <div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
-    {% if g.host %}
-    <a href="/hosts/{{ g.host.id }}" style="font-weight:600;font-size:0.95rem;color:inherit;text-decoration:none;">{{ g.host.name }}</a>
-    {% else %}
-    <span style="font-weight:600;font-size:0.95rem;">{{ g.host_name }}</span>
-    {% endif %}
-    <span style="font-size:0.78rem;color:var(--text-muted);">
-      {{ g.running }} running{% if g.stopped %} · {{ g.stopped }} stopped{% endif %}
-    </span>
-  </div>
+{# ── Needs attention ──────────────────────────────────────────────────────── #}
+{% if attention or failing_services %}
+<section class="card problem-card" aria-labelledby="docker-problems">
+  <h2 id="docker-problems" class="section-title" style="margin-bottom:0.6rem;">Needs attention</h2>
+  <ul class="problem-list">
+    {# A swarm service in trouble: Swarm replaces its failed tasks, so the
+       service, not any one container, is what is failing. #}
+    {% for s in failing_services %}
+    {% set e = s.last_error %}
+    <li>
+      {% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
+      {% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}
+      {% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
+      <a href="/plugins/docker/service/{{ s.name }}">{{ s.name }}</a>
+      <span class="problem-detail">
+        {{- s.running }} of {{ s.desired }} running
+        {%- if s.crash_looping %} · {{ s.failures_1h }} tasks failed in the last hour{% endif %}
+        {%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.exit_code %} (exit {{ e.exit_code }}){% endif %}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
+        {%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
+      </span>
+      <a class="field-hint" href="/plugins/docker/service/{{ s.name }}{% if e and e.task_id %}?task={{ e.task_id }}{% endif %}#logs" style="margin-left:0.4rem;">Logs →</a>
+    </li>
+    {% endfor %}
+    {% for r in attention %}
+    {% set c = r.container %}
+    {% set worst = r.problems[0] %}
+    <li>
+      {{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
+      <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }}">{{ r.label }}</a>
+      <span class="problem-detail">
+        {%- for p in r.problems %}{{ says[p] }}{{ " and " if not loop.last }}{% endfor %}
+        {%- if c.exit_code and worst in ('exited_nonzero', 'crash_looping', 'restarting') %} · exit {{ c.exit_code }}{% endif %}
+        {%- if r.since %} · since {{ r.since|ago }}{% endif %}
+        {%- if r.stale %} · {{ status_badge('stale') }} last report {{ c.scraped_at|ago if c.scraped_at else "never" }}{% endif -%}
+      </span>
+      <a class="field-hint" href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}/logs" style="margin-left:0.4rem;">Logs →</a>
+    </li>
+    {% endfor %}
+  </ul>
+</section>
+{% endif %}
+
+{# ── Hosts whose Docker is not answering (agent ≥ 1.9.0) ──────────────────── #}
+{% if docker_down %}
+<section class="card problem-card" aria-labelledby="docker-unreachable">
+  <h2 id="docker-unreachable" class="section-title" style="margin-bottom:0.6rem;">Docker not answering</h2>
+  <ul class="problem-list">
+    {% for d in docker_down %}
+    <li>
+      {{ status_badge('crit', 'Error') if d.status == 'error' else status_badge('stale', 'No Docker') }}
+      <a href="/hosts/{{ d.host_id }}">{{ d.host_name }}</a>
+      <span class="problem-detail">{{ d.detail or "no reason given" }} · since {{ d.since|ago }} · its containers below are what Docker last reported</span>
+    </li>
+    {% endfor %}
+  </ul>
+</section>
+{% endif %}
+
+{# ── What failed in the last 24h: what the dashboard's "failed (24h)" counts.
+   A plain container's death, or a swarm service's failed tasks; a container
+   someone stopped (exit 137/143) is not a failure. Each links to the logs of
+   what died: a service's to its failed task's lines (#5300). ───────────── #}
+{% if failures %}
+<section id="failures" class="card" aria-labelledby="docker-failures">
+  <h2 id="docker-failures" class="section-title" style="margin-bottom:0.6rem;">Failed in the last 24h</h2>
+  <ul class="problem-list">
+    {% for x in failures %}
+    <li>
+      {{ status_badge('warn', 'Service' if x.kind == 'service' else 'Container') }}
+      {% if x.kind == 'service' %}<a href="/plugins/docker/service/{{ x.name }}">{{ x.name }}</a>
+      {% else %}<a href="/plugins/docker/container/{{ x.host_id }}/{{ x.name }}">{{ x.name }}</a>{% endif %}
+      <span class="problem-detail">
+        {{- x.count }} {{ "failure" if x.count == 1 else "failures" }}
+        {%- if x.detail %} · last: {{ x.detail }}{% endif %}
+        {%- if x.host %} · {{ x.host }}{% endif %}
+        {%- if x.at %} · {{ x.at|ago }}{% endif -%}
+      </span>
+      {% if x.kind == 'service' %}
+      <a class="field-hint" href="/plugins/docker/service/{{ x.name }}{% if x.task_id %}?task={{ x.task_id }}{% endif %}#logs" style="margin-left:0.4rem;">Logs{% if x.task_id %} of that task{% endif %} →</a>
+      {% else %}
+      <a class="field-hint" href="/plugins/docker/container/{{ x.host_id }}/{{ x.name }}/logs" style="margin-left:0.4rem;">Logs →</a>
+      {% endif %}
+    </li>
+    {% endfor %}
+  </ul>
+</section>
+{% endif %}
+
+{# ── Swarm services: one row each, problems first unless a header picked
+   another order; CPU and memory add up the running replicas (#5299). ───── #}
+{% if services %}
+<section class="card-flush" aria-labelledby="docker-services">
+  <h2 id="docker-services" class="table-caption">Swarm services <span class="tone-muted">{{ services|length }}{% if services|length != services_total %} of {{ services_total }}{% endif %}</span>
+    <a href="/plugins/docker/swarm" class="field-hint">By node →</a></h2>
   <table class="table">
     <thead>
       <tr>
-        <th>Container</th>
-        <th>Image</th>
-        <th>Ports</th>
-        <th>CPU %</th>
-        <th style="min-width:100px;">CPU history</th>
-        <th>Mem %</th>
-        <th style="min-width:100px;">Mem history</th>
+        {{ th("name", "Service") }}
+        {{ th("status", "Status", title="Problems first: a crash loop, then down, then short of replicas, then failed tasks in 24h") }}
+        {{ th("stack", "Stack", title="The part of the service's name before the first _: the stack it was deployed in") }}
+        <th title="The nodes running its replicas">Nodes</th>
+        {{ th("cpu", "CPU", "num", "Across its running replicas; Docker counts CPU per core") }}
+        {{ th("mem", "Memory", "num", "Across its running replicas") }}
+        {{ th("up", "Up", "num", "Since its newest replica started") }}
+        <th class="num" title="Tasks that failed in the last 24h; Swarm replaced them">Failed 24h</th>
       </tr>
     </thead>
     <tbody>
-      {% for sub in g.subgroups %}
-      {% if g.grouped and sub.label %}
-      <tr><td colspan="7" style="padding:0.4rem 0.75rem 0.2rem;font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;font-weight:600;">{{ sub.label }}</td></tr>
-      {% endif %}
-      {% for item in sub.containers %}
-      {% set c = item.container %}
+      {% for r in services %}
       <tr>
+        <td class="cell-name"><a href="/plugins/docker/service/{{ r.name }}" title="{{ r.name }}{% if r.image %} · {{ r.image }}{% endif %}">{{ r.name }}</a></td>
         <td>
-          <div style="display:flex;align-items:center;gap:0.5rem;">
-            <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-            <div>
-              <div style="font-weight:500;font-size:0.9rem;">
-                <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="color:inherit;text-decoration:none;border-bottom:1px dotted var(--border-mid);">{{ c.name }}</a>
-                {% if c.health == 'healthy' %}<span title="healthy" style="color:var(--green);font-size:0.7rem;">●</span>
-                {% elif c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);font-size:0.7rem;">●</span>
-                {% elif c.health == 'starting' %}<span title="health: starting" style="color:var(--orange);font-size:0.7rem;">◐</span>{% endif %}
-              </div>
-              <div style="font-size:0.73rem;color:var(--text-muted);">
-                {{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
-                {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
-                  · <span style="color:var(--red);">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
-                {% endif %}
-                {% if c.restart_count %} · <span title="restart count" style="color:var(--orange);">⟳{{ c.restart_count }}</span>{% endif %}
-              </div>
-              {% if c.service_name or c.compose_project %}
-              <div style="font-size:0.68rem;color:var(--text-dim);margin-top:0.1rem;">
-                {{ c.service_name or c.compose_project }}
-              </div>
-              {% endif %}
-            </div>
-          </div>
+          {{ status_dot('up' if r.state == 'healthy' else ('warn' if r.state == 'degraded' else 'down')) }}
+          <span class="tone-{{ 'ok' if r.state == 'healthy' else ('warn' if r.state == 'degraded' else 'crit') }}" style="font-variant-numeric:tabular-nums;">{{ r.running }}/{{ r.desired }}</span>
+          {%- if r.crash_looping %} {{ status_badge('crit', 'Crash-looping') }}{% elif r.unhealthy %} {{ status_badge('crit', 'Unhealthy') }}{% endif %}
+          {%- if r.mode == 'global' %} <span class="tone-dim">global</span>{% endif %}
         </td>
-        <td style="font-size:0.82rem;color:var(--text-muted);max-width:200px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
-          {{ c.image }}
+        <td>{% if r.stack %}<a class="chip" href="{{ page_url(stack=r.stack) }}" title="Show the {{ r.stack }} stack">{{ r.stack }}</a>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="tone-muted">
+          {{- r.nodes|join(", ") or ("" if r.ghost_nodes else "—") -}}
+          {%- if r.ghost_nodes %}{{ ", " if r.nodes }}<span title="Running on a node with no Steward agent: no figures for it">{{ r.ghost_nodes|join(", ") }} (no agent)</span>{% endif -%}
         </td>
-        <td style="font-size:0.78rem;font-family:ui-monospace,monospace;">
-          {% for p in item.ports %}
-          <div style="color:var(--text-muted);">{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}</div>
-          {% endfor %}
+        <td class="num">{% if r.cpu_cores is not none %}{{ cores(r.cpu_cores) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="num">{% if r.mem_bytes is not none %}{{ r.mem_bytes|bytes }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="num">{{ up(r) }}</td>
+        <td class="num">
+          {%- if r.failures_24h %}<a class="tone-warn" href="/plugins/docker/service/{{ r.name }}#tasks"
+               title="{% if r.last_error %}Last: {{ r.last_error.error or r.last_error.message or r.last_error.state }}{% if r.last_error.exit_code %} (exit {{ r.last_error.exit_code }}){% endif %}{% endif %}">{{ r.failures_24h }}</a>
+          {%- else %}<span class="tone-dim">—</span>{% endif -%}
         </td>
-        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
-          {% if c.cpu_pct is not none %}
-          <span style="color:{% if c.cpu_pct > 80 %}var(--red){% elif c.cpu_pct > 50 %}var(--orange){% else %}var(--text){% endif %};">
-            {{ "%.1f" | format(c.cpu_pct) }}%
-          </span>
-          {% else %}
-          <span style="color:var(--text-dim);">—</span>
-          {% endif %}
-        </td>
-        <td>{{ item.sparkline_cpu | safe }}</td>
-        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
-          {% if c.mem_pct is not none %}
-          <span style="color:{% if c.mem_pct > 90 %}var(--red){% elif c.mem_pct > 70 %}var(--orange){% else %}var(--text){% endif %};">
-            {{ "%.1f" | format(c.mem_pct) }}%
-          </span>
-          {% else %}
-          <span style="color:var(--text-dim);">—</span>
-          {% endif %}
-        </td>
-        <td>{{ item.sparkline_mem | safe }}</td>
       </tr>
       {% endfor %}
-      {% endfor %}
     </tbody>
   </table>
-</div>
-{% endfor %}
-{% elif not swarm_services %}
-<div class="card" style="text-align:center;padding:3rem;">
-  <div style="color:var(--text-muted);font-size:0.9rem;">
+</section>
+{% endif %}
+
+{# ── Containers outside any swarm service, every host in one table (#5301);
+   a compose project is its stack. ─────────────────────────────────────── #}
+{% if containers %}
+<section class="card-flush" aria-labelledby="docker-containers">
+  <h2 id="docker-containers" class="table-caption">{{ "Other containers" if services_total else "Containers" }} <span class="tone-muted">{{ containers|length }}{% if containers|length != containers_total %} of {{ containers_total }}{% endif %}</span></h2>
+  <table class="table">
+    <thead>
+      <tr>
+        {{ th("name", "Container") }}
+        {{ th("status", "Status", title="Problems first: unhealthy or restarting, then died with a failing exit, then stale; then running, then stopped") }}
+        <th>Host</th>
+        {{ th("stack", "Stack", title="Its compose project") }}
+        <th>Image</th>
+        {{ th("cpu", "CPU", "num", "Docker counts CPU per core: 1 core = one core fully busy") }}
+        {{ th("mem", "Memory", "num", "Used, and the limit when one is set") }}
+        {{ th("up", "Up", "num", "Since it started") }}
+      </tr>
+    </thead>
+    <tbody>
+      {% for r in containers %}
+      <tr>
+        <td class="cell-name">
+          <a href="/plugins/docker/container/{{ r.host_id }}/{{ r.container_name }}" title="{{ r.container_name }}">{{ r.name }}</a>
+          {%- if r.ports %}<div class="cell-sub mono">{% for p in r.ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{{ " " if not loop.last }}{% endfor %}</div>{% endif %}
+        </td>
+        <td>
+          {{ status_dot('stale' if r.stale else ('up' if r.running else ('pending' if r.status == 'paused' else 'down'))) }}
+          {%- if r.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}
+          {%- elif r.health == 'starting' %} {{ status_badge('pending', 'Starting') }}
+          {%- else %} <span class="tone-muted">{{ r.status }}</span>{% endif %}
+          {%- if not r.running and r.exit_code is not none and r.exit_code != 0 %}
+          <span class="tone-{{ 'crit' if r.exit_failed else 'dim' }}" title="{{ 'last exit code' if r.exit_failed else 'stopped by a signal; not a failure' }}">exit {{ r.exit_code }}{% if r.oom_killed %} (OOM){% endif %}</span>
+          {%- endif %}
+          {%- if r.restart_count %} <span class="tone-warn" title="Restarted {{ r.restart_count }} times">⟳{{ r.restart_count }}</span>{% endif %}
+          {%- if r.stale %} {{ status_badge('stale', title="Last report " ~ (r.scraped_at.strftime('%Y-%m-%d %H:%M UTC') if r.scraped_at else 'never') ~ "; what is shown may no longer be true") }}{% endif %}
+        </td>
+        <td class="tone-muted">{{ r.host }}</td>
+        <td>{% if r.stack %}<a class="chip" href="{{ page_url(stack=r.stack) }}" title="Show the {{ r.stack }} stack">{{ r.stack }}</a>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="cell-image" title="{{ r.image }}">{{ r.image }}</td>
+        <td class="num">{% if r.running %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="num">{% if r.running %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="num">{{ up(r) }}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</section>
+{% endif %}
+
+{% if not services and not containers %}
+<div class="card">
+  {% if services_total or containers_total %}
+  <p class="tone-muted" style="margin:0;">Nothing matches these filters. <a href="/plugins/docker/">Show everything</a></p>
+  {% else %}
+  <p class="tone-muted" style="margin:0;">
     No containers reported yet. Containers are collected by the Steward host agent —
     deploy the agent to a host running Docker (Hosts → the host → agent panel) and
-    its containers will appear here under that host.
-  </div>
+    its containers will appear here.
+  </p>
+  {% endif %}
 </div>
 {% endif %}

plugins/docker/templates/docker/service.html

@@ -0,0 +1,185 @@
+{# docker/service.html — one swarm service (#5302): its state and what it
+   runs, where its replicas are and what they use, Swarm's record of its recent
+   tasks, its CPU and memory over time, and the logs of every task it has run.
+   A failure's "Logs →" lands on #logs with ?task= picked. #}
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs, status_badge, status_dot, stat_tile %}
+{% from "docker/_usage.html" import cpu, mem, cores %}
+{% block title %}{{ name }} — Docker — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
+{% block content %}
+
+{% if service is none %}
+<div class="card empty-state">
+  <strong>{{ "No longer running" if task_choices else "Service not found" }}</strong>
+  {% if task_choices %}
+  <span>No swarm service named <code>{{ name }}</code> is reported now; it may have been removed.
+    The logs Steward still keeps from its tasks are below.</span>
+  {% else %}
+  <span>No swarm service named <code>{{ name }}</code> is reported, and Steward holds no logs or
+    events for one. It may have been removed. <a href="/plugins/docker/">Back to Docker</a></span>
+  {% endif %}
+</div>
+{% else %}
+
+{# ── Header: is it well, then what it is ─────────────────────────────────── #}
+{% set tone = 'ok' if row.state == 'healthy' else ('warn' if row.state == 'degraded' else 'crit') %}
+<div class="page-header">
+  <div>
+    <h1 class="page-title" style="overflow-wrap:anywhere;">{{ name }}</h1>
+    <p class="page-subtitle">
+      {% if row.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
+      {% elif row.state == 'down' %}{{ status_badge('crit', 'Down') }}
+      {% elif row.state == 'degraded' %}{{ status_badge('warn', 'Degraded') }}
+      {% else %}{{ status_badge('ok', 'Healthy') }}{% endif %}
+      <span class="tone-{{ tone }}">{{ row.running }} of {{ row.desired }} running</span>
+      · {{ row.mode }}
+      {%- if stack %} · stack <a class="chip" href="/plugins/docker/?stack={{ stack }}">{{ stack }}</a>{% endif %}
+      {%- if row.trouble_since %} · in trouble since {{ row.trouble_since|ago }}{% endif %}
+    </p>
+  </div>
+  <div class="page-actions"><a class="btn btn-sm btn-ghost" href="#logs">Logs</a></div>
+</div>
+
+{% if row.last_error %}
+{% set e = row.last_error %}
+<section class="card problem-card" aria-labelledby="svc-last-error">
+  <h2 id="svc-last-error" class="section-title" style="margin-bottom:0.4rem;">Last failed task</h2>
+  <p style="margin:0;">
+    {{ e.error or e.message or e.state }}{% if e.exit_code %} <span class="tone-crit">(exit {{ e.exit_code }})</span>{% endif %}
+    {%- if e.node %} · on {{ e.node }}{% endif %}{% if e.age_s is not none %} · {{ e.age_s|duration }} ago{% endif %}
+    {%- if e.task_id %} · <a href="?task={{ e.task_id }}#logs">its logs →</a>{% endif %}
+  </p>
+</section>
+{% endif %}
+
+<div class="stat-strip">
+  {{ stat_tile("CPU", cores(row.cpu_cores) if row.cpu_cores is not none else "—", hint="across running replicas") }}
+  {{ stat_tile("Memory", row.mem_bytes|bytes if row.mem_bytes is not none else "—", hint="across running replicas") }}
+  {{ stat_tile("Up", (row.started_at|ago|striptags|replace(" ago", "")) if row.started_at else "—", hint="since its newest replica started") }}
+  {{ stat_tile("Failed (24h)", row.failures_24h or 0, state="warn" if row.failures_24h else none, hint="tasks Swarm replaced", href="#tasks" if row.failures_24h else none) }}
+</div>
+
+<div class="card">
+  <dl class="kv">
+    <dt>Image</dt><dd class="mono" style="overflow-wrap:anywhere;">{{ (svc.image if svc and svc.image else row.image) or "—" }}</dd>
+    <dt>Mode</dt><dd>{{ row.mode }}{% if row.mode != 'global' %}, {{ row.desired }} replica{{ "s" if row.desired != 1 }}{% endif %}</dd>
+    {% if svc %}<dt>Reported</dt><dd>{{ svc.updated_at|ago }} by the manager's agent</dd>{% endif %}
+  </dl>
+</div>
+
+{# ── Replicas: where it runs and what each uses ──────────────────────────── #}
+<section class="card-flush" aria-labelledby="svc-replicas">
+  <h2 id="svc-replicas" class="table-caption">Replicas <span class="tone-muted">{{ replicas|length }}</span></h2>
+  {% if replicas %}
+  <table class="table">
+    <thead><tr>
+      <th>Node</th><th>Container</th><th>Status</th>
+      <th class="num" title="Docker counts CPU per core">CPU</th><th class="num">Memory</th><th class="num">Up</th>
+    </tr></thead>
+    <tbody>
+      {% for r in replicas %}
+      <tr>
+        {% if r.ghost %}
+        <td>{{ r.host }}</td>
+        <td colspan="5" class="tone-dim">{{ r.count }} running on a node with no Steward agent: no figures for {{ "it" if r.count == 1 else "them" }}</td>
+        {% else %}
+        <td>{% if r.host_id %}<a href="/hosts/{{ r.host_id }}">{{ r.host }}</a>{% else %}{{ r.host }}{% endif %}</td>
+        <td class="cell-name"><a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" title="{{ r.name }}">{{ r.name }}</a></td>
+        <td>
+          {{ status_dot('up' if r.status == 'running' else 'down') }} <span class="tone-muted">{{ r.status }}</span>
+          {%- if r.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}{% elif r.health == 'starting' %} {{ status_badge('pending', 'Starting') }}{% endif %}
+        </td>
+        <td class="num">{% if r.usage %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="num">{% if r.usage %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td class="num">{{ r.uptime or "—" }}</td>
+        {% endif %}
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p class="tone-muted" style="margin:0;padding:1rem;">No replica is running.</p>
+  {% endif %}
+</section>
+
+{# ── Resource history, summed across replicas ────────────────────────────── #}
+<section class="card" aria-labelledby="svc-history">
+  <div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
+    <h2 id="svc-history" class="section-title" style="margin:0;">Resource history</h2>
+    {% include "_time_range.html" %}
+  </div>
+  <div hx-get="/plugins/docker/service/{{ name }}/history"
+       hx-trigger="load, rangeChange from:body"
+       hx-include="#time-range"
+       hx-swap="innerHTML">
+    <p class="field-hint">Loading…</p>
+  </div>
+</section>
+
+{# ── Swarm's record of its recent tasks ──────────────────────────────────── #}
+<section id="tasks" class="card-flush" aria-labelledby="svc-tasks">
+  <h2 id="svc-tasks" class="table-caption">Tasks <span class="tone-muted">Swarm's most recent, newest first</span></h2>
+  {% if tasks %}
+  <table class="table">
+    <thead><tr>
+      <th title="The replica slot this task filled">Slot</th><th>Task</th><th>Node</th><th>State</th>
+      <th class="num">Exit</th><th>Why</th><th class="num">When</th>
+    </tr></thead>
+    <tbody>
+      {% for t in tasks %}
+      <tr>
+        <td>{{ t.slot or "—" }}</td>
+        <td class="mono" title="{{ t.task_id }}">{% if t.task_id %}<a href="?task={{ t.task_id }}#logs">{{ t.task_id[:10] }}</a>{% else %}—{% endif %}</td>
+        <td>{{ t.node }}</td>
+        <td>
+          {%- if t.failed %}{{ status_badge('crit', t.state|capitalize) }}
+          {%- elif t.current %}{{ status_badge('ok', 'Running') }}
+          {%- else %}<span class="tone-muted">{{ t.state }}</span>{% endif %}
+          {%- if t.desired and t.desired != t.state and not t.current %} <span class="tone-dim" title="What Swarm wants of it">→ {{ t.desired }}</span>{% endif -%}
+        </td>
+        <td class="num">{% if t.exit_code %}<span class="tone-{{ 'crit' if t.failed else 'dim' }}">{{ t.exit_code }}</span>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+        <td style="overflow-wrap:anywhere;">{{ t.why or "" }}</td>
+        <td class="num">{% if t.at %}{{ t.at|ago }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p class="tone-muted" style="margin:0;padding:1rem;">No task record yet: the manager's agent reports them from version 1.8.0.</p>
+  {% endif %}
+</section>
+
+{% endif %}
+
+{# ── Logs across every task, newest first ─────────────────────────────────
+   Shown for a removed service too, while its lines are still kept. #}
+{% if service is not none or task_choices %}
+<section id="logs" class="card-flush" aria-labelledby="svc-logs">
+  <h2 id="svc-logs" class="table-caption">Logs <span class="tone-muted">every task, newest first</span></h2>
+  <form id="svc-log-controls" class="filter-bar" onsubmit="return false;" style="padding:0.75rem 1rem 0;">
+    <input type="search" name="q" class="input" placeholder="Filter lines…" autocomplete="off" aria-label="Filter log lines">
+    {% if task_choices|length > 1 %}
+    <select name="task" aria-label="Task">
+      <option value="">All tasks</option>
+      {% for tid, label in task_choices %}<option value="{{ tid }}"{% if tid == focus_task %} selected{% endif %}>{{ label }}</option>{% endfor %}
+    </select>
+    {% elif focus_task %}<input type="hidden" name="task" value="{{ focus_task }}">{% endif %}
+    <select name="stream" aria-label="Stream">
+      <option value="all">stdout and stderr</option>
+      <option value="stdout">stdout</option>
+      <option value="stderr">stderr</option>
+    </select>
+  </form>
+  <div id="svc-log-lines"
+       hx-get="/plugins/docker/service/{{ name }}/logs/lines"
+       hx-trigger="load, every 5s, change from:#svc-log-controls, keyup changed delay:400ms from:#svc-log-controls"
+       hx-include="#svc-log-controls"
+       hx-swap="innerHTML"
+       role="log" aria-live="polite" tabindex="0"
+       style="max-height:60vh;overflow:auto;padding:0.5rem 1rem 0.75rem;font-family:var(--fs-font-mono);font-size:0.8rem;line-height:1.5;">
+    <p class="field-hint">Loading…</p>
+  </div>
+</section>
+{% endif %}
+{% endblock %}

plugins/docker/templates/docker/swarm.html

@@ -1,87 +1,103 @@
-{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement #}
+{# docker/swarm.html — the Swarm laid out by node (#5303). A verdict, the
+   services short of replicas, then one card per node: a node that is down or
+   not taking work first, then managers. Each card says what its containers use
+   against what the machine has, and lists the services placed on it. #}
 {% extends "base.html" %}
-{% from "_macros.html" import crumbs %}
+{% from "_macros.html" import crumbs, status_badge, status_dot %}
+{% from "docker/_usage.html" import cores as fmt_cores %}
 {% block title %}Swarm — Docker — Steward{% endblock %}
 {% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Swarm", "")]) }}{% endblock %}
 {% block content %}
 
-<h1 class="page-title" style="margin-bottom:1.5rem;">Swarm</h1>
+<h1 class="page-title">Swarm</h1>
 
 {% if swarms %}
 {% for g in swarms %}
-<div style="margin-bottom:2rem;">
-  <div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.75rem;flex-wrap:wrap;">
-    <span style="font-weight:600;font-size:0.95rem;">Swarm</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);">
-      reported by {{ g.managers | length }} manager{{ 's' if g.managers | length != 1 }} · {{ g.managers | join(", ") }}
-    </span>
-  </div>
+{% set down_nodes = g.nodes|rejectattr("ok")|list %}
+<section class="swarm-group" aria-label="Swarm reported by {{ g.managers|join(', ') }}">
+  <p class="docker-verdict">
+    {% if down_nodes or g.troubled %}<strong class="tone-crit">{{ down_nodes|length + g.troubled|length }} need{{ "s" if (down_nodes|length + g.troubled|length) == 1 }} attention</strong>
+    {% else %}<strong class="tone-ok">Every node is up and taking work</strong>{% endif %}
+    · {{ g.nodes|length }} node{{ "s" if g.nodes|length != 1 }}
+    · <span class="tone-{{ 'ok' if g.services_healthy == g.services_total else 'warn' }}">{{ g.services_healthy }}/{{ g.services_total }}</span> services healthy
+    <span class="tone-dim">· reported by {{ g.managers|join(", ") }}</span>
+  </p>
 
-  {# ── Services ─────────────────────────────────────────────────────────── #}
-  <div class="card-flush" style="margin-bottom:1rem;">
-    <table class="table">
-      <thead>
-        <tr>
-          <th>Service</th><th>Mode</th><th>Replicas</th><th>Image</th><th>Placement</th>
-        </tr>
-      </thead>
-      <tbody>
-        {% for s in g.services %}
-        <tr>
-          <td style="font-weight:500;font-size:0.9rem;">{{ s.name }}</td>
-          <td style="font-size:0.82rem;color:var(--text-muted);">{{ s.mode }}</td>
-          <td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
-            <span style="color:{% if s.healthy %}var(--green){% elif s.down %}var(--red){% elif s.degraded %}var(--orange){% else %}var(--text-muted){% endif %};">
-              {{ s.running }}/{{ s.desired }}
-            </span>
-          </td>
-          <td style="font-size:0.8rem;color:var(--text-muted);max-width:220px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.image or "—" }}</td>
-          <td style="font-size:0.78rem;color:var(--text-muted);">
-            {% for p in s.placement %}{{ p.node }} ({{ p.running }}){% if not loop.last %}, {% endif %}{% else %}—{% endfor %}
-          </td>
-        </tr>
-        {% else %}
-        <tr><td colspan="5" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No services in this swarm.</td></tr>
-        {% endfor %}
-      </tbody>
-    </table>
-  </div>
+  {% if down_nodes or g.troubled %}
+  <section class="card problem-card" aria-label="Needs attention">
+    <ul class="problem-list">
+      {% for n in down_nodes %}
+      <li>
+        {{ status_badge('crit', 'Node ' ~ n.status) if n.status != 'ready' else status_badge('warn', n.availability|capitalize) }}
+        <span>{{ n.hostname }}</span>
+        <span class="problem-detail">{{ n.role }} · {{ n.task_count }} task{{ "s" if n.task_count != 1 }} placed on it</span>
+      </li>
+      {% endfor %}
+      {% for s in g.troubled %}
+      <li>
+        {% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}{% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}{% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
+        <a href="/plugins/docker/service/{{ s.name }}">{{ s.name }}</a>
+        {% set e = s.last_error %}
+        <span class="problem-detail">
+          {{- s.running }} of {{ s.desired }} running
+          {%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
+          {%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
+        </span>
+      </li>
+      {% endfor %}
+    </ul>
+  </section>
+  {% endif %}
 
-  {# ── Nodes ────────────────────────────────────────────────────────────── #}
-  <div class="card-flush">
-    <table class="table">
-      <thead>
-        <tr><th>Node</th><th>Role</th><th>Availability</th><th>Status</th></tr>
-      </thead>
-      <tbody>
-        {% for n in g.nodes %}
-        <tr>
-          <td style="font-weight:500;font-size:0.9rem;">
-            {{ n.hostname or n.node_id }}
-            {% if n.leader %}<span title="cluster leader" style="font-size:0.68rem;color:var(--accent);border:1px solid var(--accent);border-radius:3px;padding:0 0.3rem;margin-left:0.3rem;">leader</span>{% endif %}
-          </td>
-          <td style="font-size:0.82rem;color:var(--text-muted);">{{ n.role }}</td>
-          <td style="font-size:0.82rem;color:{% if n.availability == 'active' %}var(--text){% else %}var(--orange){% endif %};">{{ n.availability }}</td>
-          <td style="font-size:0.82rem;">
-            <span class="dot {% if n.status == 'ready' %}dot-up{% else %}dot-down{% endif %}"></span>
-            <span style="color:{% if n.status == 'ready' %}var(--green){% else %}var(--red){% endif %};">{{ n.status }}</span>
-          </td>
-        </tr>
+  <div class="node-grid">
+    {% for n in g.nodes %}
+    <section class="card node-card{% if not n.ok %} problem-card{% endif %}" aria-labelledby="node-{{ n.node_id }}">
+      <h2 id="node-{{ n.node_id }}" class="node-card-head">
+        {{ status_dot('up' if n.status == 'ready' else 'down') }}
+        {% if n.host_id %}<a href="/hosts/{{ n.host_id }}">{{ n.hostname }}</a>{% else %}{{ n.hostname }}{% endif %}
+        <span class="chip">{{ "leader" if n.leader else n.role }}</span>
+        {% if n.availability != 'active' %}{{ status_badge('warn', n.availability|capitalize, title="Swarm places no new tasks here") }}{% endif %}
+      </h2>
+      <p class="node-card-figures">
+        {% if n.cores_used is not none %}
+        Containers use <strong>{{ fmt_cores(n.cores_used) }}</strong>{% if n.cores %} of {{ n.cores }}{% endif %}
+        · <strong>{{ n.mem_used|bytes }}</strong>{% if n.mem %} of {{ n.mem|bytes }}{% endif %}
+        {% elif n.agent %}
+        <span class="tone-muted">No swarm task reports figures here yet.</span>
         {% else %}
-        <tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No nodes reported.</td></tr>
-        {% endfor %}
-      </tbody>
-    </table>
+        <span class="tone-muted" title="Install the Steward agent on this node to see what its tasks use">No Steward agent: what runs here is Swarm's placement, without figures.</span>
+        {% endif %}
+      </p>
+      {% if n.tasks %}
+      <table class="table node-tasks">
+        <thead><tr><th>Service</th><th class="num">Tasks</th><th class="num">CPU</th><th class="num">Memory</th></tr></thead>
+        <tbody>
+          {% for t in n.tasks %}
+          <tr>
+            <td class="cell-name">
+              {%- if t.state != 'healthy' %}{{ status_dot('down' if t.state == 'down' else 'warn') }} {% endif -%}
+              <a href="/plugins/docker/service/{{ t.name }}" title="{{ t.name }}">{{ t.name }}</a>
+            </td>
+            <td class="num">{{ t.tasks }}</td>
+            <td class="num">{% if t.cores is not none %}{{ fmt_cores(t.cores) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+            <td class="num">{% if t.mem_bytes is not none %}{{ t.mem_bytes|bytes }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
+          </tr>
+          {% endfor %}
+        </tbody>
+      </table>
+      {% else %}
+      <p class="tone-muted" style="margin:0;">No tasks placed here.</p>
+      {% endif %}
+    </section>
+    {% endfor %}
   </div>
-</div>
+</section>
 {% endfor %}
 {% else %}
-<div class="card" style="text-align:center;padding:3rem;">
-  <div style="color:var(--text-muted);font-size:0.9rem;">
-    No Swarm topology reported. A Steward host agent running on a Swarm
-    <strong>manager</strong> reports services, nodes, and task placement here —
-    workers and non-swarm hosts contribute only their local containers.
-  </div>
+<div class="card empty-state">
+  No Swarm topology reported. A Steward host agent running on a Swarm
+  <strong>manager</strong> reports services, nodes and task placement here;
+  workers and hosts outside a swarm contribute only their own containers.
 </div>
 {% endif %}
 {% endblock %}

plugins/docker/templates/docker/widget.html

@@ -1,67 +1,80 @@
-{# docker/widget.html — dashboard widget: container status overview, by host #}
+{# docker/widget.html — the dashboard Docker widget (#5252), in the widget-shape
+   order: a verdict line; the services and containers in trouble; what failed
+   in the last 24h, by name; then everything healthy, one line each, packed
+   into as many columns as the widget is wide. Resource use is the Busiest
+   Containers widget's, so it isn't repeated here. #}
+{% from "_macros.html" import status_dot %}
 {% if total_count == 0 %}
-<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No containers reported yet.</p>
+<div class="w-verdict" style="color:var(--text-muted);">No containers reported yet.</div>
 {% else %}
-
-<div style="display:flex;gap:1.25rem;margin-bottom:0.65rem;">
-  <div>
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--green);">{{ running_count }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">running</span>
-  </div>
-  <div title="Distinct containers with a die or OOM event in the last 24h">
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:{% if failed_24h %}var(--red){% else %}var(--text-muted){% endif %};">{{ failed_24h }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">failed (24h)</span>
-  </div>
+{% set wrong = bad_services|length + bad_containers|length %}
+<div class="w-verdict {{ 'tone-crit' if wrong else ('tone-warn' if failures else 'tone-ok') }}">
+  {%- if services_total %}{{ services_healthy }}/{{ services_total }} services healthy · {% endif -%}
+  {{ running_count }} other container{{ 's' if running_count != 1 }} running ·
+  {% if failures %}<a href="/plugins/docker/#failures" style="color:inherit;">{{ failures|length }} failed in 24h</a>{% else %}nothing failed in 24h{% endif -%}
 </div>
 
-{# Swarm services — collapsed, each with its replicas' host chips (dashed = a
-   node with no Steward agent, count-only from placement). #}
-{% if swarm_services %}
-<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">Swarm services</div>
-<div style="display:grid;gap:0.4rem;margin-bottom:0.4rem;">
-  {% for s in swarm_services %}
-  <div>
-    <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
-      <span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
-      <span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
-      <span style="color:var(--text-muted);font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;">{{ s.running }}/{{ s.desired }}</span>
-    </div>
-    <div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">
-      {% for r in s.replicas %}
-      {% if r.ghost %}
-      <span title="no Steward agent on this node — count from swarm placement" style="font-size:0.68rem;color:var(--text-dim);background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }} ×{{ r.count }}</span>
-      {% else %}
-      <span style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }}</span>
-      {% endif %}
-      {% endfor %}
-    </div>
-  </div>
-  {% endfor %}
-</div>
-{% endif %}
-
-{% for g in host_groups %}
-{% if multi_host %}
-<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">{{ g.host_name }}</div>
-{% endif %}
-<div style="display:grid;gap:2px;">
-  {% for c in g.containers %}
-  <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
-    <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-    <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;">
-      {{ c.name }}{% if c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);"> ●</span>{% elif c.health == 'healthy' %}<span title="healthy" style="color:var(--green);"> ●</span>{% endif %}{% if c.restart_count %}<span title="restarts" style="color:var(--orange);font-size:0.7rem;"> ⟳{{ c.restart_count }}</span>{% endif %}
-    </a>
-    {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
-    <span title="last exit code" style="font-size:0.7rem;color:var(--red);flex-shrink:0;">exit {{ c.exit_code }}</span>
-    {% endif %}
-    {% if c.cpu_pct is not none %}
-    <span style="font-size:0.72rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
-      {{ "%.1f" | format(c.cpu_pct) }}%
+{% if wrong %}
+<div class="w-section">Needs a look</div>
+{% for s in bad_services %}
+<div class="w-item">
+  {{ status_dot('degraded' if s.state == 'degraded' else 'down') }}
+  <span class="w-label" title="{{ s.name }}">{{ s.name }}
+    <span class="w-sub {{ 'tone-crit' if s.state != 'degraded' else 'tone-warn' }}">
+      {%- if s.crash_looping %}crash-looping: {{ s.failures_1h }} failed in the last hour{% else %}{{ s.running }} of {{ s.desired }} running{% endif %}
+      {%- if s.trouble_since %}, since {{ s.trouble_since|ago }}{% endif %}
+      {%- if s.last_error and (s.last_error.error or s.last_error.message) %} · {{ s.last_error.error or s.last_error.message }}{% endif -%}
     </span>
-    {% endif %}
-  </div>
-  {% endfor %}
+  </span>
+  <span class="w-meta" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
+</div>
+{% endfor %}
+{% for c in bad_containers %}
+<div class="w-item">
+  {{ status_dot('warn' if c.status == 'restarting' else 'down') }}
+  <span class="w-label" title="{{ c.name }}">
+    {%- if session.user_id %}<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>{% else %}{{ c.name }}{% endif %}
+    <span class="w-sub tone-crit">
+      {%- if c.health == 'unhealthy' %}unhealthy{% elif c.status == 'restarting' %}restarting{% else %}exited {{ c.exit_code }}{% if c.oom_killed %}, out of memory{% endif %}{% endif %} · {{ host_name.get(c.host_id, c.host_id) }}
+    </span>
+  </span>
 </div>
 {% endfor %}
+{% endif %}
 
+{% if failures %}
+<div class="w-section">Failed in the last 24h</div>
+{% for f in failures[:5] %}
+<div class="w-item">
+  {{ status_dot('failed') }}
+  <span class="w-label" title="{{ f.name }}">{{ f.name }}
+    <span class="w-sub">{% if f.count > 1 %}{{ f.count }} times, last {% endif %}{{ f.at|ago }}{% if f.detail %} · {{ f.detail }}{% endif %} · {{ f.host }}</span>
+  </span>
+</div>
+{% endfor %}
+{% if failures|length > 5 %}<div class="w-more">+{{ failures|length - 5 }} more · <a href="/plugins/docker/#failures">the full list</a></div>{% endif %}
+{% endif %}
+
+{% if healthy %}
+<div class="w-section">Healthy</div>
+<div class="w-grid">
+  {% for h in healthy %}
+  <div class="w-item">
+    {% if h.kind == 'service' %}{% set s = h.s %}
+    {{ status_dot('up') }}
+    <span class="w-label" title="{{ s.name }} — on {{ s.replicas|map(attribute='host')|unique|join(', ') }}">{{ s.name }}
+      <span style="color:var(--text-dim);"> · {{ s.replicas|map(attribute='host')|unique|join(', ') }}</span></span>
+    <span class="w-meta" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
+    {% else %}{% set c = h.c %}
+    {{ status_dot('up' if h.kind == 'container' else 'unknown') }}
+    <span class="w-label" title="{{ c.name }}">
+      {%- if session.user_id %}<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>{% else %}{{ c.name }}{% endif %}
+      <span style="color:var(--text-dim);"> · {{ host_name.get(c.host_id, c.host_id) }}</span></span>
+    {% if h.kind == 'stopped' %}<span class="w-meta">stopped</span>{% endif %}
+    {% endif %}
+  </div>
+  {% endfor %}
+</div>
+{% if healthy_more %}<div class="w-more">+{{ healthy_more }} more healthy · <a href="/plugins/docker/">all of Docker</a></div>{% endif %}
+{% endif %}
 {% endif %}

plugins/docker/templates/docker/widget_resources.html

@@ -1,29 +1,51 @@
-{# docker/widget_resources.html — dashboard widget: the busiest running containers
-   by CPU, each with labeled CPU + memory utilisation bars. #}
+{# docker/widget_resources.html — the dashboard Busiest Containers widget
+   (#5253, #5214): a verdict naming the busiest, then the top containers by CPU
+   (averaged over the last few minutes, so the order holds still). Each figure
+   carries its unit, beside bars of the last few hours. A row's bars are scaled
+   to its own peak, but never below a floor (1 core, 1 GiB by default), so a
+   container idling at 0.05 cores reads flat and a spike means real load. #}
+{% macro cores(v) -%}{{ "%.2f"|format(v) if v < 1 else "%.1f"|format(v) }}{%- endmacro %}
+{% macro bars(b, peak_text, roof_text) -%}
+<span class="w-bars w-wide-only" aria-hidden="true"
+      title="{% if b.peak is none %}No samples in the last {{ hours }}h{% else %}Peak {{ peak_text }} in the last {{ hours }}h; bars reach the top at {{ roof_text }}{% endif %}">
+  {%- for h in b.bars %}<i{% if h is none %} class="gap"{% else %} style="height:{{ [h, 4]|max }}%"{% endif %}></i>{% endfor -%}
+</span>
+{%- endmacro %}
 {% if not rows %}
-<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No running containers.</div>
+<div class="w-verdict" style="color:var(--text-muted);">No running containers.</div>
 {% else %}
-{% macro bar(label, pct, warn, crit) %}
-<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;">
-  <span style="font-size:0.62rem;color:var(--text-dim);width:1.9rem;flex-shrink:0;letter-spacing:0.03em;">{{ label }}</span>
-  <div style="flex:1;height:6px;background:var(--bg-elevated);border-radius:3px;overflow:hidden;">
-    <div style="height:100%;border-radius:3px;width:{{ [pct, 100] | min }}%;
-                background:{% if pct > crit %}var(--red){% elif pct > warn %}var(--orange){% else %}var(--accent){% endif %};
-                transition:width 0.4s;"></div>
-  </div>
-  <span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;width:3rem;text-align:right;flex-shrink:0;">{{ "%.1f" | format(pct) }}%</span>
+{% set top = rows[0] %}
+<div class="w-verdict">
+  Busiest: <strong>{{ top.label }}</strong> on {{ top.host_name }},
+  {% if top.usage.cores is not none %}{{ cores(top.usage.cores) }}
+  {%- if top.usage.cpus %} of {{ top.usage.cpus }} cores{% else %} cores{% endif %}{% else %}no CPU reading yet{% endif %}.
 </div>
-{% endmacro %}
-<div style="display:grid;gap:0.6rem;">
+<table class="w-table">
+  <colgroup><col><col class="w-wide-only" style="width:7.5rem"><col class="w-barcol"><col class="w-barcol"></colgroup>
+  <thead><tr><th>Container</th><th class="w-wide-only w-text">Host</th>
+    <th title="Cores, averaged over the last {{ minutes }} minutes; bars: the last {{ hours }}h">CPU<span class="w-wide-only"> · last {{ hours }}h</span></th>
+    <th title="In use now; bars: the last {{ hours }}h">Memory<span class="w-wide-only"> · last {{ hours }}h</span></th></tr></thead>
+  <tbody>
   {% for r in rows %}
-  {% set c = r.c %}
-  <div>
-    <div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;">
-      {{ c.name }}{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
-    </div>
-    {% if c.cpu_pct is not none %}{{ bar("CPU", c.cpu_pct, 50, 80) }}{% endif %}
-    {% if c.mem_pct is not none %}{{ bar("MEM", c.mem_pct, 70, 90) }}{% endif %}
-  </div>
+  {% set u = r.usage %}
+  <tr>
+    <td class="w-name" title="{{ r.c.name }}">
+      {%- if session.user_id %}<a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}">{{ r.label }}</a>{% else %}{{ r.label }}{% endif -%}
+    </td>
+    <td class="w-wide-only w-text" title="{{ r.host_name }}">{{ r.host_name }}</td>
+    <td class="tone-{{ status_class(u.cpu_share, 'cpu') if u.cpus and u.cores is not none else 'dim' }}"
+        {% if u.cpus and u.cores is not none %}title="{{ '%.1f'|format(u.cpu_share) }}% of {{ r.host_name }}'s {{ u.cpus }} cores"{% endif %}>
+      {{- bars(r.cpu_bars, (cores(r.cpu_bars.peak) ~ " cores") if r.cpu_bars.peak is not none else "", cores(r.cpu_bars.roof) ~ " cores") -}}
+      <span class="w-fig">{% if u.cores is none %}—{% else %}{{ cores(u.cores) }} cores{% endif %}</span>
+    </td>
+    <td class="tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of and u.mem_bytes is not none else 'dim' }}"
+        {% if u.mem_pct is not none and u.mem_of and u.mem_bytes is not none %}title="{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}"{% endif %}>
+      {{- bars(r.mem_bars, r.mem_bars.peak|bytes if r.mem_bars.peak is not none else "", r.mem_bars.roof|bytes) -}}
+      <span class="w-fig">{% if u.mem_bytes is none %}—{% else %}{{ u.mem_bytes|bytes }}{% endif %}</span>
+    </td>
+  </tr>
   {% endfor %}
-</div>
+  </tbody>
+</table>
+<div class="w-note w-wide-only">Bars: {{ (hours * 60 // 24) }}-minute slices, oldest left; each row up to its own peak, at least {{ cores(cpu_floor) }} core{{ 's' if cpu_floor != 1 }} and {{ mem_floor|bytes }}.</div>
 {% endif %}

plugins/host_agent/__init__.py

@@ -21,3 +21,24 @@ def get_scheduled_tasks() -> list:
 def get_blueprint():
     from .routes import host_agent_bp
     return host_agent_bp
+
+
+def register_mcp_tools(register) -> None:
+    from .mcp_tools import register_tools
+    register_tools(register)
+
+
+async def mcp_overview(db) -> list[dict]:
+    from .mcp_tools import mcp_overview as findings
+    return await findings(db)
+
+
+def get_alert_metrics() -> dict[str, list[str]]:
+    """Host metrics ingest records for alert rules (resource = the host name;
+    disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
+    comes from the scheduler, not the agent: "agent offline" is > 180."""
+    return {"host_agent": [
+        "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
+        "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m",
+        "uptime_secs", "seconds_since_seen",
+    ]}

plugins/host_agent/agent.py

@@ -17,10 +17,10 @@ import time
 import urllib.error
 import urllib.request
 from collections import deque
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.6.0"
+AGENT_VERSION = "1.11.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -34,7 +34,10 @@ class ConfigError(Exception):
 
 REQUIRED_KEYS = ("url", "token")
 INT_KEYS = ("interval_seconds",)
-LIST_KEYS = ("mounts",)
+LIST_KEYS = ("mounts", "docker_log_exclude")
+# Truthy strings for bool keys; anything else (incl. empty) is False.
+BOOL_KEYS = ("docker_logs_enabled",)
+_BOOL_TRUE = ("1", "true", "yes", "on")
 
 
 def read_config(path: str) -> dict:
@@ -58,6 +61,8 @@ def read_config(path: str) -> dict:
                         raise ConfigError(f"{path}:{lineno}: {key} must be int")
                 elif key in LIST_KEYS:
                     cfg[key] = [v.strip() for v in value.split(",") if v.strip()]
+                elif key in BOOL_KEYS:
+                    cfg[key] = value.lower() in _BOOL_TRUE
                 else:
                     cfg[key] = value
     except FileNotFoundError:
@@ -69,6 +74,10 @@ def read_config(path: str) -> dict:
 
     cfg.setdefault("interval_seconds", 30)
     cfg.setdefault("docker_socket", DEFAULT_DOCKER_SOCKET)
+    # Container-log collection is on by default (operator preference); a host can
+    # opt out with `docker_logs_enabled = false` or thin it with docker_log_exclude.
+    cfg.setdefault("docker_logs_enabled", True)
+    cfg.setdefault("docker_log_exclude", [])
     return cfg
 
 
@@ -82,6 +91,10 @@ OS_RELEASE_PATH = "/etc/os-release"
 NETDEV_PATH = "/proc/net/dev"
 DISKSTATS_PATH = "/proc/diskstats"
 HWMON_DIR = "/sys/class/hwmon"
+# Where the kernel lists virtual network devices (bridges, veth, docker0,
+# vxlan, tun, VLANs, bonds) and the block devices (a disk has a `device`).
+SYS_VIRTUAL_NET_DIR = "/sys/devices/virtual/net"
+SYS_BLOCK_DIR = "/sys/block"
 PSI_DIR = "/proc/pressure"
 
 # Partition names to drop from disk I/O (we report whole-disk throughput only).
@@ -250,10 +263,34 @@ def default_mounts() -> list[str]:
     return mounts or ["/"]
 
 
+def _physical_only(found: dict, is_virtual) -> dict:
+    """The entries the kernel says are physical, so a host total counts each
+    byte once (a bridge's or an LVM volume's traffic is also counted on the
+    NIC or disk beneath it). When /sys calls every one of them virtual (an
+    agent inside LXC, where eth0 is a veth), they are all kept: a host
+    reporting no traffic at all would be wrong, not filtered."""
+    physical = {k: v for k, v in found.items() if not is_virtual(k)}
+    return physical or found
+
+
+def _virtual_net(name: str) -> bool:
+    """Whether the kernel lists this interface as virtual; False if /sys can't say."""
+    return os.path.isdir(SYS_VIRTUAL_NET_DIR) and os.path.exists(
+        os.path.join(SYS_VIRTUAL_NET_DIR, name))
+
+
+def _virtual_disk(name: str) -> bool:
+    """Whether this block device has no disk behind it (dm, md, zram, nbd);
+    False if /sys can't say."""
+    return os.path.isdir(SYS_BLOCK_DIR) and not os.path.exists(
+        os.path.join(SYS_BLOCK_DIR, name, "device"))
+
+
 def collect_net_raw() -> dict:
     """Return {iface: (rx_bytes, tx_bytes)} cumulative counters from /proc/net/dev.
 
-    Skips loopback and veth* (container) interfaces to cut noise. Counters are
+    Physical interfaces only (see _physical_only); loopback and veth* are
+    skipped by name too, for a host whose /sys can't be read. Counters are
     monotonic; per-second rates are derived from deltas in build_sample().
     """
     out: dict = {}
@@ -275,14 +312,15 @@ def collect_net_raw() -> dict:
             out[name] = (int(f[0]), int(f[8]))  # rx_bytes, tx_bytes
         except ValueError:
             continue
-    return out
+    return _physical_only(out, _virtual_net)
 
 
 def collect_diskio_raw() -> dict:
     """Return {device: (read_bytes, write_bytes)} cumulative from /proc/diskstats.
 
-    Whole disks only — loop/ram/sr/fd and partitions are skipped. Sectors are
-    512 bytes. Counters are monotonic; rates derived from deltas.
+    Physical whole disks only (see _physical_only): loop/ram/sr/fd and
+    partitions are skipped by name too. Sectors are 512 bytes. Counters are
+    monotonic; rates derived from deltas.
     """
     out: dict = {}
     try:
@@ -300,7 +338,7 @@ def collect_diskio_raw() -> dict:
             out[name] = (int(f[5]) * 512, int(f[9]) * 512)  # sectors_read, sectors_written
         except ValueError:
             continue
-    return out
+    return _physical_only(out, _virtual_disk)
 
 
 def collect_temps() -> list:
@@ -402,6 +440,24 @@ def _rates(cur: dict, prev: dict, dt: float) -> dict:
 # when that header is present rather than assume Content-Length.
 
 DOCKER_API_TIMEOUT = 5.0
+# All of one sample's per-container Docker work (inspect + stats) must finish
+# inside this; past it the sample says Docker did not answer in time rather
+# than holding up the host's own metrics (#5154).
+DOCKER_DEADLINE_SECONDS = 20.0
+# /system/df walks every image, container and volume: asked this often, not
+# every sample. Steward keeps the last answer between reports.
+DOCKER_DF_INTERVAL_SECONDS = 600.0
+
+# Container-log collection (m79). Logs ride in the same push as metrics, so one
+# interval's batch is capped to keep a chatty container from bloating a POST (and
+# the backoff ring): once the cap is hit a marker line is emitted and collection
+# stops — the deferred lines come on the next interval. On a container's first
+# interval we seed from a short tail, then switch to an incremental since-cursor
+# kept per container in the agent's rate-state.
+DOCKER_LOG_BATCH_MAX_BYTES = 262144   # 256 KiB of log text per push, all containers
+DOCKER_LOG_FIRST_TAIL = 50            # lines seeded on a container's first interval
+DOCKER_LOG_FINAL_TAIL = 200           # lines read once from a container found already crashed
+DOCKER_LOG_FINAL_WINDOW_SECONDS = 600  # ... when it stopped this recently
 
 
 def _dechunk(body: bytes) -> bytes:
@@ -545,6 +601,18 @@ def _docker_inspect_fields(insp: dict):
     return health, restart_count, exit_code, oom_killed
 
 
+def _docker_started_at(insp: dict, key: str = "StartedAt"):
+    """State.StartedAt as ISO-8601, or None for a container never started
+    (Docker's zero time) or an inspect that failed. When the container last
+    started, not when it was created: a restarted container's uptime counts
+    from its restart. key="FinishedAt" reads when it last stopped."""
+    raw = (((insp or {}).get("State") or {}).get(key) or "")
+    if not raw or raw.startswith("0001-"):
+        return None
+    ts = _parse_log_ts(raw)
+    return ts.isoformat() if ts else None
+
+
 def _docker_net_io(stats: dict):
     """Return cumulative (net_rx, net_tx, blk_read, blk_write) bytes from stats.
 
@@ -578,7 +646,7 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
 
     # Inspect every container (incl. stopped) — exit codes + restart counts only
     # live here, and stopped containers are exactly where exit_code matters.
-    health = exit_code = None
+    health = exit_code = insp = None
     restart_count = 0
     oom_killed = False
     try:
@@ -598,11 +666,10 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
         except (OSError, ValueError):
             pass
 
-    created = c.get("Created")
-    started_at = (
-        datetime.fromtimestamp(created, tz=timezone.utc).isoformat()
-        if isinstance(created, (int, float)) else None
-    )
+    started_at = _docker_started_at(insp)
+    # When it last stopped (a running container's is its previous stop, or
+    # None): dates a die event, and tells the log collector a stop is new.
+    finished_at = None if state == "running" else _docker_started_at(insp, "FinishedAt")
     record = {
         "name": name,
         "container_id": cid[:12],
@@ -614,6 +681,7 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
         "mem_pct": mem_pct,
         "ports": _docker_ports(c.get("Ports", [])),
         "started_at": started_at,
+        "finished_at": finished_at,
         "health": health,
         "restart_count": restart_count,
         "exit_code": exit_code,
@@ -627,27 +695,276 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
     return record
 
 
-def collect_docker(socket_path: str) -> list:
-    """Per-container state from the local Docker socket, or [] if unavailable.
+def _docker_failure(socket_path: str, exc: Exception) -> tuple:
+    """(status, detail) for a failed Docker call: "unavailable" when there is no
+    Docker here to ask, "error" when there is one and the call went wrong."""
+    if isinstance(exc, FileNotFoundError):
+        return "unavailable", "no Docker socket at " + socket_path
+    if isinstance(exc, ConnectionRefusedError):
+        return "unavailable", "Docker is not accepting connections on " + socket_path
+    if isinstance(exc, PermissionError):
+        return "error", "permission denied on " + socket_path
+    if isinstance(exc, socket.timeout):
+        return "error", "Docker did not answer within %.0f s" % DOCKER_API_TIMEOUT
+    return "error", (str(exc) or type(exc).__name__)[:200]
 
-    Absent socket / permission denied / non-Docker endpoint all yield [] — the
-    agent silently reports no containers rather than erroring, so a Docker-less
-    host needs no configuration.
+
+def collect_docker(socket_path: str) -> tuple:
+    """(status, detail, containers) from the local Docker socket.
+
+    status is "ok" when Docker answered (containers may then be [] on a host
+    with none, which tells Steward to clear its list), "unavailable" when there
+    is no Docker here (no socket, or nothing listening), and "error" when Docker
+    is here but the listing failed. detail says why, for the last two. A
+    Docker-less host still needs no configuration: it reports "unavailable".
     """
     try:
         containers = _docker_request(socket_path, "/containers/json?all=true")
-    except (OSError, ValueError):
-        return []
-    if not isinstance(containers, list) or not containers:
-        return []
+    except (OSError, ValueError) as exc:
+        return _docker_failure(socket_path, exc) + ([],)
+    if not isinstance(containers, list):
+        return "error", "unexpected answer from /containers/json", []
+    if not containers:
+        return "ok", None, []
 
     # Each container needs an inspect (+ a stats read if running), and the stats
     # call blocks ~1s while the daemon computes the cpu delta. Done serially that
     # stretches the whole sample on a busy host, so fan the per-container work out
     # over a small bounded thread pool (I/O-bound → threads are enough).
-    workers = min(8, len(containers))
-    with ThreadPoolExecutor(max_workers=workers) as ex:
-        return list(ex.map(lambda c: _collect_one_container(socket_path, c), containers))
+    # A deadline on the whole fan-out: one wedged container must not stall the
+    # sample. Unfinished work is reported as Docker not answering in time, not
+    # as a partial list: a container left out would read as removed, and one
+    # without its inspect as restarted or changed health.
+    ex = ThreadPoolExecutor(max_workers=min(8, len(containers)))
+    try:
+        futures = [ex.submit(_collect_one_container, socket_path, c) for c in containers]
+        done, pending = wait_futures(futures, timeout=DOCKER_DEADLINE_SECONDS)
+    finally:
+        try:
+            ex.shutdown(wait=False, cancel_futures=True)
+        except TypeError:          # Python < 3.9: no cancel_futures
+            ex.shutdown(wait=False)
+    if pending:
+        return ("error", "%d of %d containers not read within %g s"
+                % (len(pending), len(containers), DOCKER_DEADLINE_SECONDS), [])
+    return "ok", None, [f.result() for f in futures]
+
+
+# ─── container logs (m79) ─────────────────────────────────────────────────────
+#
+# The logs endpoint returns a raw multiplexed byte stream, not JSON, so it needs
+# a sibling of _docker_request that hands back bytes. Non-TTY containers frame
+# stdout/stderr with an 8-byte header (stream byte + 4-byte big-endian length);
+# TTY containers emit unframed bytes. We request timestamps=1, so every line is
+# prefixed with an RFC3339Nano timestamp we parse for the since-cursor.
+
+_LOG_STREAM_NAMES = {0: "stdout", 1: "stdout", 2: "stderr"}
+
+
+def _docker_request_raw(socket_path: str, path: str,
+                        timeout: float = DOCKER_API_TIMEOUT) -> bytes:
+    """GET `path` from the Docker API over the Unix socket; return the raw body.
+
+    Sibling of `_docker_request` for non-JSON endpoints (container logs). Reuses
+    the connect / send / de-chunk scaffolding; raises OSError on any transport
+    problem or non-2xx status so callers can silent-skip. A 200 with an empty
+    body (no new lines since the cursor) returns b"".
+    """
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.settimeout(timeout)
+    try:
+        sock.connect(socket_path)
+        req = (
+            "GET " + path + " HTTP/1.1\r\n"
+            "Host: docker\r\n"
+            "Accept: application/octet-stream\r\n"
+            "Connection: close\r\n"
+            "\r\n"
+        )
+        sock.sendall(req.encode("ascii"))
+        chunks = []
+        while True:
+            buf = sock.recv(65536)
+            if not buf:
+                break
+            chunks.append(buf)
+    finally:
+        sock.close()
+
+    raw = b"".join(chunks)
+    head, _, body = raw.partition(b"\r\n\r\n")
+    header_text = head.decode("latin-1")
+    status_line = header_text.split("\r\n", 1)[0]
+    parts = status_line.split(None, 2)
+    status = int(parts[1]) if len(parts) >= 2 and parts[1].isdigit() else 0
+    if not (200 <= status < 300):
+        raise OSError(f"docker API {path} returned {status}")
+    if "transfer-encoding: chunked" in header_text.lower():
+        body = _dechunk(body)
+    return body
+
+
+def _demux_docker_logs(raw: bytes) -> list:
+    """Split a Docker logs stream into [(stream_name, payload_bytes), …].
+
+    Non-TTY containers multiplex with an 8-byte frame header (stream byte in
+    {0,1,2} + 4-byte big-endian length). The stream is treated as framed only if
+    the whole buffer parses as clean back-to-back frames; anything else (a TTY
+    container's raw stream, or a malformed header) falls back to a single stdout
+    blob so no bytes are lost.
+    """
+    if not raw:
+        return []
+    frames = []
+    i, n = 0, len(raw)
+    while i + 8 <= n:
+        stream = raw[i]
+        if stream > 2 or raw[i + 1] or raw[i + 2] or raw[i + 3]:
+            break  # not a valid frame header → not framed
+        size = int.from_bytes(raw[i + 4:i + 8], "big")
+        if i + 8 + size > n:
+            break  # frame overruns the buffer → not framed
+        frames.append((_LOG_STREAM_NAMES.get(stream, "stdout"),
+                       raw[i + 8:i + 8 + size]))
+        i += 8 + size
+    if frames and i == n:
+        return frames
+    return [("stdout", raw)]
+
+
+def _parse_log_ts(token: str):
+    """Parse a Docker RFC3339Nano timestamp token into a datetime, or None.
+
+    Normalises `Z` → `+00:00` and trims Docker's nanosecond precision to the
+    microseconds datetime.fromisoformat accepts (stdlib, Python 3.8+ safe).
+    """
+    t = token.strip()
+    if not t:
+        return None
+    if t.endswith("Z"):
+        t = t[:-1] + "+00:00"
+    m = re.match(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(\.\d+)?(.*)$", t)
+    if m:
+        frac = m.group(2) or ""
+        if len(frac) > 7:               # "." + up to 6 fractional digits
+            frac = frac[:7]
+        t = m.group(1) + frac + m.group(3)
+    try:
+        return datetime.fromisoformat(t)
+    except ValueError:
+        return None
+
+
+def _parse_container_logs(raw: bytes) -> list:
+    """Demux + line-split a logs stream → [(dt|None, stream, line), …] in order.
+
+    Each line is timestamp-prefixed (we request timestamps=1); an unparseable
+    prefix yields dt=None with the full raw line kept.
+    """
+    out = []
+    for stream, payload in _demux_docker_logs(raw):
+        text = payload.decode("utf-8", "replace")
+        for raw_line in text.split("\n"):
+            if not raw_line:
+                continue
+            token, _, msg = raw_line.partition(" ")
+            dt = _parse_log_ts(token)
+            if dt is not None:
+                out.append((dt, stream, msg))
+            else:
+                out.append((None, stream, raw_line))
+    return out
+
+
+def collect_docker_logs(socket_path: str, containers: list, state: dict,
+                        exclude=None,
+                        max_bytes: int = DOCKER_LOG_BATCH_MAX_BYTES,
+                        now: float | None = None) -> list:
+    """New container-log lines since the last interval.
+
+    Keeps a per-container since-cursor in `state["docker_log_cursors"]`
+    (name → unix ts) so each interval fetches only what's new; a container's
+    first interval seeds from a short tail. A running container is read every
+    interval. A stopped one is read once per stop (`finished_at`, remembered in
+    `state["docker_log_finals"]`), so the lines it wrote on its way down reach
+    Steward (#5049): since its cursor if it was being read, or, for one first
+    seen already stopped with a non-zero exit in the last
+    DOCKER_LOG_FINAL_WINDOW_SECONDS, its last DOCKER_LOG_FINAL_TAIL lines. That
+    is how a task that crashes between two samples is explained by its own
+    output. Both maps forget a container once it is no longer listed.
+
+    The whole batch is capped at `max_bytes` of log text — once exceeded a
+    truncation marker is appended and collection stops (logs must never balloon
+    a push). `exclude` container names are skipped entirely (a per-host
+    bandwidth opt-out). Best-effort: a per-container transport error just
+    contributes nothing this interval.
+    Returns [{"container", "stream", "ts", "line"}, …].
+    """
+    exclude_set = set(exclude or ())
+    cursors = state.setdefault("docker_log_cursors", {})
+    finals = state.setdefault("docker_log_finals", {})
+    now = time.time() if now is None else now
+    listed = {c.get("name"): c for c in (containers or []) if c.get("name")}
+    # Forget containers no longer listed so state can't grow without bound
+    # over the agent's lifetime.
+    for book in (cursors, finals):
+        for gone in [k for k in book if k not in listed]:
+            del book[gone]
+
+    out: list = []
+    total = 0
+    truncated = False
+    for name, c in listed.items():
+        if name in exclude_set:
+            continue
+        prev = cursors.get(name)
+        base = f"/containers/{name}/logs?stdout=1&stderr=1&timestamps=1"
+        if c.get("status") == "running":
+            # since accepts a fractional Unix ts; we still dedup the boundary line.
+            path = (f"{base}&tail={DOCKER_LOG_FIRST_TAIL}" if prev is None
+                    else f"{base}&since={prev:.9f}")
+        else:
+            stopped = c.get("finished_at")
+            if not stopped or finals.get(name) == stopped:
+                continue
+            if prev is not None:
+                path = f"{base}&since={prev:.9f}"
+            else:
+                at = _parse_log_ts(stopped)
+                if (c.get("exit_code") in (None, 0) or at is None
+                        or now - at.timestamp() > DOCKER_LOG_FINAL_WINDOW_SECONDS):
+                    continue
+                path = f"{base}&tail={DOCKER_LOG_FINAL_TAIL}"
+            finals[name] = stopped
+        try:
+            raw = _docker_request_raw(socket_path, path)
+        except (OSError, ValueError):
+            continue
+        newest = prev
+        for dt, stream, line in _parse_container_logs(raw):
+            ts_unix = dt.timestamp() if dt is not None else None
+            if prev is not None and ts_unix is not None and ts_unix <= prev:
+                continue  # boundary line already sent last interval
+            if total >= max_bytes:
+                truncated = True
+                break
+            out.append({"container": name, "stream": stream,
+                        "ts": dt.isoformat() if dt is not None else None,
+                        "line": line})
+            total += len(line)
+            if ts_unix is not None and (newest is None or ts_unix > newest):
+                newest = ts_unix
+        if newest is not None:
+            cursors[name] = newest
+        if truncated:
+            break
+
+    if truncated:
+        out.append({"container": "_steward", "stream": "stderr",
+                    "ts": datetime.now(timezone.utc).isoformat(),
+                    "line": (f"[steward] log batch truncated at {max_bytes} bytes; "
+                             "remaining lines deferred to the next interval")})
+    return out
 
 
 # ─── swarm (manager-only) ────────────────────────────────────────────────────
@@ -664,14 +981,46 @@ def _swarm_is_manager(info: dict) -> bool:
     return bool((info or {}).get("Swarm", {}).get("ControlAvailable"))
 
 
+# Newest tasks kept per service. Swarm retains about 5 per slot by default, so
+# this covers a replicated service's recent history and a crash loop's deaths
+# without shipping a global service's whole fleet every sample.
+SWARM_TASKS_PER_SERVICE = 10
+_TASK_TEXT_CHARS = 300
+
+
+def _swarm_task(t: dict) -> dict:
+    """One task as Steward stores it: where it ran, what Swarm wants of it,
+    what it is doing, and why it stopped (Err/Message, exit code)."""
+    status = t.get("Status") or {}
+    cstatus = status.get("ContainerStatus") or {}
+    exit_code = cstatus.get("ExitCode")
+    return {
+        "task_id": t.get("ID", "") or "",
+        "slot": t.get("Slot"),
+        "node_id": t.get("NodeID", "") or "",
+        "desired_state": (t.get("DesiredState") or "").lower(),
+        "state": (status.get("State") or "").lower(),
+        "message": (status.get("Message") or "")[:_TASK_TEXT_CHARS],
+        "error": (status.get("Err") or "")[:_TASK_TEXT_CHARS],
+        "exit_code": exit_code if isinstance(exit_code, int) else None,
+        "at": status.get("Timestamp") or t.get("UpdatedAt") or "",
+    }
+
+
 def _swarm_services(services: list, tasks: list) -> list:
     """Roll desired-vs-running replicas per service up from the task list.
 
     Replica health isn't on the service object — it's derived by counting tasks:
     `running` = tasks currently in state running; `desired` = the service's
     configured replica count (replicated) or the count of tasks the scheduler
-    wants running (global, which has no fixed number).
+    wants running (global, which has no fixed number). Each service also
+    carries its newest tasks (`tasks`), so Steward can tell a task Swarm
+    replaced from one that crashed, and say why the last one failed.
     """
+    tasks_by_svc: dict = {}
+    for t in (tasks or []):
+        if t.get("ServiceID"):
+            tasks_by_svc.setdefault(t["ServiceID"], []).append(_swarm_task(t))
     running_by_svc: dict = {}
     desired_by_svc: dict = {}  # only used for global-mode services
     placement: dict = {}       # svc_id -> {node_id: running_count}
@@ -711,6 +1060,8 @@ def _swarm_services(services: list, tasks: list) -> list:
             # sees every task, even ones whose containers live on other hosts).
             "placement": [{"node_id": n, "running": cnt}
                           for n, cnt in sorted(placement.get(sid, {}).items())],
+            "tasks": sorted(tasks_by_svc.get(sid, []), key=lambda t: t["at"],
+                            reverse=True)[:SWARM_TASKS_PER_SERVICE],
         })
     return out
 
@@ -734,26 +1085,28 @@ def _swarm_nodes(nodes: list) -> list:
     return out
 
 
-def collect_swarm(socket_path: str):
-    """Swarm topology from a manager node, or None on workers / non-swarm hosts.
+def collect_swarm(socket_path: str) -> tuple:
+    """(is_manager, topology) from the local Docker socket.
 
     Self-detects manager via `/info` (one cheap call); only then queries the
-    manager-only endpoints. Any transport failure degrades to None, same silent
-    contract as collect_docker — a non-swarm host adds nothing to the payload.
+    manager-only endpoints. is_manager is None when /info itself failed, so a
+    transient error is never mistaken for a node that stopped being a manager;
+    False tells Steward to drop the swarm rows this host reported as one.
+    topology is None unless this is a manager and every call answered.
     """
     try:
         info = _docker_request(socket_path, "/info")
     except (OSError, ValueError):
-        return None
+        return None, None
     if not _swarm_is_manager(info):
-        return None
+        return False, None
     try:
         services = _docker_request(socket_path, "/services")
         tasks = _docker_request(socket_path, "/tasks")
         nodes = _docker_request(socket_path, "/nodes")
     except (OSError, ValueError):
-        return None
-    return {
+        return True, None
+    return True, {
         "services": _swarm_services(
             services if isinstance(services, list) else [],
             tasks if isinstance(tasks, list) else []),
@@ -852,13 +1205,17 @@ class RingBuffer:
 
 
 def build_sample(mounts: list[str], state: dict,
-                 docker_socket: str = DEFAULT_DOCKER_SOCKET) -> dict:
+                 docker_socket: str = DEFAULT_DOCKER_SOCKET, *,
+                 docker_logs_enabled: bool = True,
+                 docker_log_exclude=None) -> dict:
     """Collect one full sample. Partial samples allowed if a collector fails.
 
     `state` carries the previous network/disk counters + monotonic timestamp so
-    throughput rates can be derived from deltas; it is mutated in place.
-    `docker_socket` is probed best-effort — the `docker` key is omitted entirely
-    when no containers are found, so non-Docker hosts add nothing to the payload.
+    throughput rates can be derived from deltas (and the per-container log
+    since-cursors); it is mutated in place. `docker_socket` is probed
+    best-effort — the `docker` key is omitted entirely when no containers are
+    found, so non-Docker hosts add nothing to the payload. `docker_logs_enabled`
+    / `docker_log_exclude` gate incremental container-log collection.
     """
     sample: dict = {"ts": datetime.now(timezone.utc).isoformat()}
     try:
@@ -913,26 +1270,33 @@ def build_sample(mounts: list[str], state: dict,
     ]
     state["net"], state["disk"], state["mono"] = net_raw, disk_raw, now_mono
 
+    # Docker's status rides every sample, so Steward can tell "no containers"
+    # (status ok, docker: []) from "couldn't ask" (unavailable / error).
     try:
-        docker = collect_docker(docker_socket)
-    except Exception:
-        docker = []
-    if docker:
+        docker_status, docker_detail, docker = collect_docker(docker_socket)
+    except Exception as exc:
+        docker_status, docker_detail, docker = "error", (str(exc) or type(exc).__name__)[:200], []
+    sample["docker_status"] = docker_status
+    if docker_detail:
+        sample["docker_detail"] = docker_detail
+    if docker_status == "ok":
         sample["docker"] = docker
 
-    # Swarm topology is only emitted by managers (collect_swarm returns None
-    # everywhere else), so the key is absent on workers / non-swarm hosts.
+    # Swarm topology is only emitted by managers. A node whose /info says it
+    # is not a manager says so, so Steward can expire what it once reported.
     try:
-        swarm = collect_swarm(docker_socket)
+        is_manager, swarm = collect_swarm(docker_socket)
     except Exception:
-        swarm = None
+        is_manager, swarm = None, None
     if swarm is not None:
         sample["swarm"] = swarm
+    elif is_manager is False:
+        sample["swarm_manager"] = False
 
-    # Image/disk usage — host-level docker fact (one /system/df call), omitted on
-    # Docker-less hosts. Only meaningful when there's docker here at all, so skip
-    # the call entirely when collect_docker found nothing.
-    if docker:
+    # Image/disk usage — host-level docker fact (one /system/df call), only
+    # when Docker answered.
+    if docker_status == "ok" and now_mono - state.get("df_mono", float("-inf")) >= DOCKER_DF_INTERVAL_SECONDS:
+        state["df_mono"] = now_mono
         try:
             disk = collect_disk_usage(docker_socket)
         except Exception:
@@ -940,6 +1304,18 @@ def build_sample(mounts: list[str], state: dict,
         if disk is not None:
             sample["docker_disk"] = disk
 
+    # Container logs (m79): incremental per-container tail folded into the same
+    # push as metrics. Skipped when there are no containers or logging is off;
+    # the key is omitted when nothing new arrived this interval.
+    if docker and docker_logs_enabled:
+        try:
+            logs = collect_docker_logs(docker_socket, docker, state,
+                                       exclude=docker_log_exclude)
+        except Exception:
+            logs = []
+        if logs:
+            sample["docker_logs"] = logs
+
     return sample
 
 
@@ -955,7 +1331,32 @@ def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict:
 # ─── POST + backoff ──────────────────────────────────────────────────────────
 
 
-BACKOFF_CAP = 300
+# Below Steward's 180 s staleness window (host_agent stale_after_seconds): once
+# Steward is reachable again a host reports within a minute, instead of reading
+# as stale for up to five (#5154; lesson: a check-in inherits its call's backoff).
+BACKOFF_CAP = 60
+# Samples in one POST: the server writes at most this many (routes.py
+# MAX_SAMPLES_PER_INGEST), so a backlog drains a few per post, oldest first,
+# rather than in one post too big to write before the timeout.
+MAX_SAMPLES_PER_POST = 5
+
+
+def _drop_logs(sample: dict) -> dict:
+    """Strip container logs from a sample before it's buffered for retry.
+
+    Logs are the one payload we never carry across a backoff — they'd bloat the
+    ring buffer and go stale — while metrics are kept so an outage doesn't lose
+    them. The since-cursor already advanced when the logs were collected, so the
+    dropped lines are simply not re-sent (accepted loss during an outage).
+    """
+    sample.pop("docker_logs", None)
+    return sample
+
+
+def split_backlog(buffered: list, limit: int = MAX_SAMPLES_PER_POST) -> tuple:
+    """(the buffered samples to send with this one, the ones held for later):
+    the oldest limit-1, leaving room for the new sample."""
+    return buffered[:limit - 1], buffered[limit - 1:]
 
 
 def next_backoff(current: int) -> int:
@@ -1022,6 +1423,8 @@ def main_loop(conf_path: str) -> int:
     hostname = cfg.get("hostname") or socket.gethostname()
     mounts = cfg.get("mounts") or default_mounts()
     docker_socket = cfg.get("docker_socket") or DEFAULT_DOCKER_SOCKET
+    docker_logs_enabled = cfg.get("docker_logs_enabled", True)
+    docker_log_exclude = cfg.get("docker_log_exclude") or []
     buffer = RingBuffer(maxlen=20)
     backoff = 0
     # Carries previous net/disk counters + monotonic ts for rate computation.
@@ -1036,24 +1439,32 @@ def main_loop(conf_path: str) -> int:
                 cfg = read_config(conf_path)
                 mounts = cfg.get("mounts") or default_mounts()
                 docker_socket = cfg.get("docker_socket") or DEFAULT_DOCKER_SOCKET
+                docker_logs_enabled = cfg.get("docker_logs_enabled", True)
+                docker_log_exclude = cfg.get("docker_log_exclude") or []
                 metadata = collect_metadata()  # refresh host_ip/distro on reload
                 _log("INFO", "config reloaded")
             except ConfigError as e:
                 _log("ERROR", f"reload failed: {e}")
             _reload_requested = False
 
-        sample = build_sample(mounts, rate_state, docker_socket)
+        sample = build_sample(mounts, rate_state, docker_socket,
+                              docker_logs_enabled=docker_logs_enabled,
+                              docker_log_exclude=docker_log_exclude)
         buffered = buffer.drain()
+        send, held = split_backlog(buffered)
         payload = build_payload(
-            samples=buffered + [sample],
+            samples=send + [sample],
             hostname=hostname,
             metadata=metadata,
         )
         ok, status = post_payload(cfg["url"], cfg["token"], payload)
 
         if ok:
-            if buffered:
-                _log("INFO", f"flushed {len(buffered)} buffered samples")
+            for s in held:
+                buffer.push(s)
+            if send:
+                _log("INFO", f"flushed {len(send)} buffered samples"
+                             + (f", {len(held)} still held" if held else ""))
             backoff = 0
             sleep_for = cfg["interval_seconds"]
         else:
@@ -1061,12 +1472,12 @@ def main_loop(conf_path: str) -> int:
                 _log("ERROR", "server rejected payload (400) — dropping sample")
             elif status == 401:
                 _log("ERROR", "token rejected (401) — check config + UI")
-                buffer.push(sample)
+                buffer.push(_drop_logs(sample))
             else:
                 _log("WARN", f"POST failed (status={status}); buffering")
                 for s in buffered:
-                    buffer.push(s)
-                buffer.push(sample)
+                    buffer.push(_drop_logs(s))
+                buffer.push(_drop_logs(sample))
             backoff = next_backoff(backoff)
             sleep_for = backoff
 

plugins/host_agent/mcp_tools.py

@@ -0,0 +1,81 @@
+"""The host_agent plugin's MCP tools and its get_overview findings."""
+from __future__ import annotations
+
+from datetime import timedelta
+
+from sqlalchemy import select
+
+from steward.hosts import service as host_svc
+from steward.mcp._context import mcp_app
+from steward.mcp.tools._shape import age_s, db_session, iso, now_utc
+from steward.mcp.tools.overview import finding
+from steward.models.hosts import Host
+from steward.models.users import UserRole
+
+from .models import HostAgentRegistration
+
+# A registration this old that has never checked in is a failed install, not one in progress.
+PENDING_GRACE = timedelta(minutes=30)
+
+
+def agent_state(reg, now, stale_after_s: int) -> str:
+    """The agent's state: reporting, stale (stopped checking in) or pending (never has)."""
+    if reg.last_seen_at is None:
+        return "pending"
+    return "reporting" if (now - reg.last_seen_at).total_seconds() <= stale_after_s else "stale"
+
+
+async def _agents(db):
+    return (await db.execute(
+        select(HostAgentRegistration, Host)
+        .join(Host, Host.id == HostAgentRegistration.host_id)
+        .order_by(Host.name)
+    )).all()
+
+
+async def list_agents() -> dict:
+    """Every host agent Steward has registered, with its version and whether it is reporting.
+
+    `state` is "reporting", "stale" (it checked in before but its last
+    check-in is older than the stale window, `stale_after_s`) or "pending"
+    (registered, never checked in, usually an install that is still running
+    or that failed). Each row carries the agent version, kernel, distro, arch
+    and the IP the agent reports. get_host shows an agent's latest metrics.
+    """
+    now = now_utc()
+    stale_after = host_svc.agent_stale_after_seconds(mcp_app().config)
+    async with db_session() as db:
+        rows = await _agents(db)
+    agents = [
+        {
+            "host_id": h.id, "host_name": h.name,
+            "state": agent_state(reg, now, stale_after),
+            "last_seen_at": iso(reg.last_seen_at), "last_seen_age_s": age_s(reg.last_seen_at, now),
+            "agent_version": reg.agent_version, "kernel": reg.kernel, "distro": reg.distro,
+            "arch": reg.arch, "reported_ip": reg.host_ip,
+            "registered_at": iso(reg.created_at),
+        }
+        for reg, h in rows
+    ]
+    counts = {s: sum(1 for a in agents if a["state"] == s) for s in ("reporting", "stale", "pending")}
+    return {"stale_after_s": stale_after, "counts": counts, "agents": agents, "total": len(agents)}
+
+
+async def mcp_overview(db) -> list[dict]:
+    """Agents registered more than PENDING_GRACE ago that have never checked in.
+
+    Agents that stopped reporting are already flagged by the core overview
+    from their metrics, so they are not repeated here.
+    """
+    now = now_utc()
+    return [
+        finding("warning", "agent_never_reported", h.name,
+                "host agent was registered but has never checked in; the install may have failed",
+                since=reg.created_at, ids={"host_id": h.id}, next="list_agents()")
+        for reg, h in await _agents(db)
+        if reg.last_seen_at is None and now - reg.created_at > PENDING_GRACE
+    ]
+
+
+def register_tools(register) -> None:
+    register(list_agents, role=UserRole.viewer, read_only=True)

plugins/host_agent/metrics_query.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 from datetime import datetime, timedelta, timezone
 
-from sqlalchemy import func, or_, select
+from sqlalchemy import func, select
 
 from steward.core.time_range import bucket_seconds
 from steward.models.metrics import PluginMetric, PluginMetricHourly
@@ -27,30 +27,14 @@ HISTORY_METRICS = (
 
 async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]:
     """{resource_name: {metric: value}} — latest sample per (resource, metric) for
-    a host + its sub-resources.
-
-    DISTINCT ON picks the newest row per group in one index-ordered pass over
-    ix_plugin_metrics_module_resource_metric_recorded, instead of a GROUP-BY-max
-    subquery self-joined back to the table (two passes over the whole history).
+    a host + its sub-resources, from plugin_metric_latest (#5208). A host whose
+    agent has gone quiet keeps its last values until they age out of the raw
+    window; the caller decides staleness from the registration's last_seen_at.
     """
-    rows = (await session.execute(
-        select(PluginMetric)
-        .where(
-            PluginMetric.source_module == SOURCE_MODULE,
-            or_(
-                PluginMetric.resource_name == host_name,
-                PluginMetric.resource_name.like(host_name + ":%"),
-            ),
-        )
-        .distinct(PluginMetric.resource_name, PluginMetric.metric_name)
-        .order_by(
-            PluginMetric.resource_name,
-            PluginMetric.metric_name,
-            PluginMetric.recorded_at.desc(),
-        )
-    )).scalars().all()
+    from steward.core.metric_series import latest_values
+
     out: dict[str, dict[str, float]] = {}
-    for r in rows:
+    for r in await latest_values(session, SOURCE_MODULE, host_name, include_children=True):
         out.setdefault(r.resource_name, {})[r.metric_name] = r.value
     return out
 

plugins/host_agent/routes.py

@@ -1,7 +1,9 @@
 # plugins/host_agent/routes.py
 from __future__ import annotations
 
+import asyncio
 import hashlib
+import time
 from datetime import datetime, timezone
 from typing import Any
 
@@ -9,11 +11,13 @@ from pathlib import Path
 
 import secrets
 from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for, session
-from steward.auth.middleware import require_role
+from steward.auth.middleware import public_route, require_role
 from steward.models.users import UserRole
-from sqlalchemy import select, func, or_, and_
+from sqlalchemy import select, or_, and_
 from datetime import timedelta
 
+from steward.core.formatting import worst_reading
+from steward.core.metric_series import points as metric_points
 from steward.core.settings import get_setting, public_base_url
 from steward.core.time_range import parse_range, RANGE_OPTIONS
 from steward.models.hosts import Host
@@ -166,6 +170,7 @@ def _error(status: int, code: str, detail: str | None = None) -> tuple[Any, int]
 
 
 @host_agent_bp.post("/ingest")
+@public_route("the agent authenticates with its bearer token")
 async def ingest():
     auth = request.headers.get("Authorization", "")
     if not auth.startswith("Bearer "):
@@ -186,110 +191,205 @@ async def ingest():
         return _error(400, "malformed_payload", "'samples' must be a non-empty list")
 
     token_hash = _hash_token(raw)
-    async with current_app.db_sessionmaker() as session:
-        reg = (await session.execute(
-            select(HostAgentRegistration).where(
-                HostAgentRegistration.token_hash == token_hash)
-        )).scalar_one_or_none()
-        if reg is None:
-            return _error(401, "invalid_token")
-        host = (await session.execute(
-            select(Host).where(Host.id == reg.host_id)
-        )).scalar_one_or_none()
-        if host is None:
-            return _error(401, "invalid_token")
+    phases = _Phases()
+    samples, dropped = newest_samples(samples, MAX_SAMPLES_PER_INGEST)
+    if dropped:
+        current_app.logger.warning(
+            "host_agent ingest: kept the newest %d of %d samples for token=%s…",
+            len(samples), len(samples) + dropped, token_hash[:8])
+    try:
+        async with current_app.db_sessionmaker() as session:
+            reg = (await session.execute(
+                select(HostAgentRegistration).where(
+                    HostAgentRegistration.token_hash == token_hash)
+            )).scalar_one_or_none()
+            if reg is None:
+                return _error(401, "invalid_token")
+            host = (await session.execute(
+                select(Host).where(Host.id == reg.host_id)
+            )).scalar_one_or_none()
+            if host is None:
+                return _error(401, "invalid_token")
 
-        accepted = 0
-        latest_ts: datetime | None = None
-        docker_snapshots: list[tuple[datetime, list]] = []
-        latest_swarm: dict | None = None
-        latest_swarm_ts: datetime | None = None
-        latest_disk: dict | None = None
-        latest_disk_ts: datetime | None = None
-        for sample in samples:
-            try:
-                recorded_at = _parse_ts(sample["ts"])
-            except (KeyError, ValueError):
-                continue
-            metrics = _expand_sample_to_metrics(sample, host.name, recorded_at)
-            for m in metrics:
-                session.add(m)
-            docker = sample.get("docker")
-            if isinstance(docker, list) and docker:
-                docker_snapshots.append((recorded_at, docker))
-            # Swarm is current-state, not time-series — keep only the newest
-            # sample's topology (a manager re-reports it every interval).
-            swarm = sample.get("swarm")
-            if isinstance(swarm, dict) and (
-                    latest_swarm_ts is None or recorded_at > latest_swarm_ts):
-                latest_swarm, latest_swarm_ts = swarm, recorded_at
-            # Disk usage is current-state too — keep only the newest sample's.
-            disk = sample.get("docker_disk")
-            if isinstance(disk, dict) and (
-                    latest_disk_ts is None or recorded_at > latest_disk_ts):
-                latest_disk, latest_disk_ts = disk, recorded_at
-            accepted += 1
-            if latest_ts is None or recorded_at > latest_ts:
-                latest_ts = recorded_at
-
-        if accepted == 0:
-            await session.rollback()
-            return _error(400, "malformed_payload", "no valid samples")
-
-        # Hand host-scoped container data to the docker plugin if it's enabled
-        # (opportunistic synergy via the capability registry — no hard import,
-        # no-op when docker is disabled). A failure here must never sink the
-        # whole ingest, so the metrics above still land.
-        if docker_snapshots or latest_swarm is not None or latest_disk is not None:
-            from steward.core.capabilities import has_capability, invoke_capability
-            if has_capability("docker.persist_host_samples"):
+            accepted = 0
+            metric_rows: list[PluginMetric] = []
+            latest_ts: datetime | None = None
+            docker_snapshots: list[tuple[datetime, list]] = []
+            docker_log_batches: list[tuple[datetime, list]] = []
+            latest_swarm: dict | None = None
+            latest_swarm_ts: datetime | None = None
+            latest_disk: dict | None = None
+            latest_disk_ts: datetime | None = None
+            # Whether Docker answered, from the newest sample that says (agent ≥ 1.9.0).
+            docker_status: dict | None = None
+            for sample in samples:
                 try:
-                    # SAVEPOINT so a docker-side failure rolls back only the
-                    # docker writes, leaving the host metrics intact to commit.
-                    async with session.begin_nested():
-                        await invoke_capability(
-                            "docker.persist_host_samples", UserRole.admin,
-                            session, host, docker_snapshots, latest_swarm,
-                            latest_disk,
-                        )
-                except Exception:
-                    current_app.logger.exception(
-                        "host_agent ingest: docker persist failed for host=%s",
-                        host.name)
+                    recorded_at = _parse_ts(sample["ts"])
+                except (KeyError, ValueError):
+                    continue
+                metric_rows.extend(_expand_sample_to_metrics(sample, host.name, recorded_at))
+                # An empty list is a real answer: Docker is up with no containers.
+                docker = sample.get("docker")
+                if isinstance(docker, list):
+                    docker_snapshots.append((recorded_at, docker))
+                status = sample.get("docker_status")
+                if status in ("ok", "unavailable", "error") and (
+                        docker_status is None or recorded_at > docker_status["at"]):
+                    docker_status = {
+                        "status": status, "detail": sample.get("docker_detail"),
+                        "at": recorded_at, "swarm_manager": sample.get("swarm_manager")}
+                # Container logs are time-series (append every line, not newest-only);
+                # each record carries its own Docker ts, recorded_at is the fallback.
+                docker_logs = sample.get("docker_logs")
+                if isinstance(docker_logs, list) and docker_logs:
+                    docker_log_batches.append((recorded_at, docker_logs))
+                # Swarm is current-state, not time-series — keep only the newest
+                # sample's topology (a manager re-reports it every interval).
+                swarm = sample.get("swarm")
+                if isinstance(swarm, dict) and (
+                        latest_swarm_ts is None or recorded_at > latest_swarm_ts):
+                    latest_swarm, latest_swarm_ts = swarm, recorded_at
+                # Disk usage is current-state too — keep only the newest sample's.
+                disk = sample.get("docker_disk")
+                if isinstance(disk, dict) and (
+                        latest_disk_ts is None or recorded_at > latest_disk_ts):
+                    latest_disk, latest_disk_ts = disk, recorded_at
+                accepted += 1
+                if latest_ts is None or recorded_at > latest_ts:
+                    latest_ts = recorded_at
 
-        md = payload.get("metadata") or {}
-        changed = False
-        if latest_ts and (reg.last_seen_at is None or latest_ts > reg.last_seen_at):
-            reg.last_seen_at = latest_ts
-            changed = True
-        version = payload.get("agent_version")
-        if version and reg.agent_version != version:
-            reg.agent_version = version
-            changed = True
-        for field in ("kernel", "distro", "arch"):
-            if field in md and getattr(reg, field) != md[field]:
-                setattr(reg, field, md[field])
+            if accepted == 0:
+                await session.rollback()
+                return _error(400, "malformed_payload", "no valid samples")
+            phases.mark("parse")
+
+            # Through the alert pipeline, so rules on host metrics can fire.
+            from steward.core.alerts import record_metrics
+            phases.size = f"samples={accepted} metric_rows={len(metric_rows)}"
+            await record_metrics(session, metric_rows, phases)
+
+            # Hand host-scoped container data to the docker plugin if it's enabled
+            # (opportunistic synergy via the capability registry — no hard import,
+            # no-op when docker is disabled). A failure here must never sink the
+            # whole ingest, so the metrics above still land.
+            if (docker_snapshots or latest_swarm is not None or docker_status is not None
+                    or latest_disk is not None or docker_log_batches):
+                from steward.core.capabilities import has_capability, invoke_capability
+                if has_capability("docker.persist_host_samples"):
+                    try:
+                        # SAVEPOINT so a docker-side failure rolls back only the
+                        # docker writes, leaving the host metrics intact to commit.
+                        async with session.begin_nested():
+                            await invoke_capability(
+                                "docker.persist_host_samples", UserRole.admin,
+                                session, host, docker_snapshots, latest_swarm,
+                                latest_disk, docker_log_batches, docker_status,
+                                phases=phases,
+                            )
+                    except Exception:
+                        current_app.logger.exception(
+                            "host_agent ingest: docker persist failed for host=%s",
+                            host.name)
+                    phases.mark("docker")
+
+            md = payload.get("metadata") or {}
+            changed = False
+            if latest_ts and (reg.last_seen_at is None or latest_ts > reg.last_seen_at):
+                reg.last_seen_at = latest_ts
                 changed = True
-        host_ip = md.get("host_ip")
-        if host_ip and reg.host_ip != host_ip:
-            reg.host_ip = host_ip
-            changed = True
-        # Mirror the reported IP into Host.address only when it's blank.
-        new_address = pick_host_address(host.address, host_ip)
-        if new_address is not None:
-            host.address = new_address
-        if changed:
-            reg.updated_at = datetime.now(timezone.utc)
+            version = payload.get("agent_version")
+            if version and reg.agent_version != version:
+                reg.agent_version = version
+                changed = True
+            for field in ("kernel", "distro", "arch"):
+                if field in md and getattr(reg, field) != md[field]:
+                    setattr(reg, field, md[field])
+                    changed = True
+            host_ip = md.get("host_ip")
+            if host_ip and reg.host_ip != host_ip:
+                reg.host_ip = host_ip
+                changed = True
+            # Mirror the reported IP into Host.address only when it's blank.
+            new_address = pick_host_address(host.address, host_ip)
+            if new_address is not None:
+                host.address = new_address
+            if changed:
+                reg.updated_at = datetime.now(timezone.utc)
 
-        if latest_ts:
-            skew = abs((datetime.now(timezone.utc) - latest_ts).total_seconds())
-            if skew > 300:
-                current_app.logger.warning(
-                    "host_agent ingest: clock skew %.0fs for host=%s", skew, host.name)
+            if latest_ts:
+                skew = abs((datetime.now(timezone.utc) - latest_ts).total_seconds())
+                if skew > 300:
+                    current_app.logger.warning(
+                        "host_agent ingest: clock skew %.0fs for host=%s", skew, host.name)
 
-        await session.commit()
+            await session.commit()
+            phases.mark("commit")
+    except asyncio.CancelledError:
+        # The agent hung up (its 10 s timeout) and the server dropped the
+        # request: say how far it got before re-raising.
+        phases.cancelled(token_hash[:8])
+        raise
 
-    return jsonify({"ok": True, "samples_accepted": accepted}), 200
+    phases.report(host.name, accepted, len(metric_rows),
+                  sum(len(c) for _, c in docker_snapshots),
+                  sum(len(lines) for _, lines in docker_log_batches))
+    return jsonify({"ok": True, "samples_accepted": accepted, "samples_dropped": dropped}), 200
+
+
+# The agent gives up on a post after 10 s (agent.py); an ingest past this
+# says where its time went, so a slow phase shows in the logs (#5149).
+SLOW_INGEST_SECONDS = 2.0
+
+# How many samples one ingest writes (#5149). An agent that could not post
+# sends its whole buffer (up to 21 samples, ~4,000 metric rows); on the live
+# instance that did not finish inside the agent's 10 s, the rollback left the
+# buffer full, and every later post failed the same way. The newest samples
+# are kept: the older ones would be lost to the agent's ring buffer anyway.
+# Five matches the agent's MAX_SAMPLES_PER_POST (1.10+), so its backlog posts
+# land whole: ~1,000 metric rows at the 1,300-4,000 rows/s measured once
+# plugin_metrics got time-ordered ids, with posts answering in 0.4-1.3 s (#5181).
+MAX_SAMPLES_PER_INGEST = 5
+
+
+def newest_samples(samples: list, limit: int) -> tuple[list, int]:
+    """(the newest `limit` samples, oldest first; how many were left out).
+
+    Ordered by their ISO-8601 `ts` (the agent writes them all in one form);
+    one without a ts sorts oldest and is rejected later as malformed."""
+    if len(samples) <= limit:
+        return samples, 0
+    ordered = sorted(samples, key=lambda s: str(s.get("ts") or "") if isinstance(s, dict) else "")
+    return ordered[-limit:], len(samples) - limit
+
+
+class _Phases:
+    """Wall-clock per phase of one ingest request."""
+
+    def __init__(self):
+        self.start = self.last = time.monotonic()
+        self.took: dict[str, float] = {}
+        self.size = ""
+
+    def mark(self, name: str) -> None:
+        now = time.monotonic()
+        self.took[name] = self.took.get(name, 0.0) + (now - self.last)
+        self.last = now
+
+    def cancelled(self, who: str) -> None:
+        current_app.logger.warning(
+            "host_agent ingest: dropped after %.1fs for token=%s… (completed: %s) %s",
+            time.monotonic() - self.start, who,
+            " ".join(f"{k}={v:.1f}s" for k, v in self.took.items()) or "nothing", self.size)
+
+    def report(self, host: str, samples: int, metrics: int, containers: int, log_lines: int) -> None:
+        total = time.monotonic() - self.start
+        if total < SLOW_INGEST_SECONDS:
+            return
+        current_app.logger.warning(
+            "host_agent ingest: slow %.1fs for host=%s (%s) samples=%d metric_rows=%d "
+            "containers=%d log_lines=%d", total, host,
+            " ".join(f"{k}={v:.1f}s" for k, v in self.took.items()),
+            samples, metrics, containers, log_lines)
 
 
 AGENT_SOURCE_PATH = Path(__file__).parent / "agent.py"
@@ -306,6 +406,7 @@ def _agent_version() -> str:
 
 
 @host_agent_bp.get("/install.sh")
+@public_route("fetched by curl on the host being enrolled; carries an enrolment token")
 async def install_script():
     token = request.args.get("token", "")
     if not token:
@@ -337,6 +438,7 @@ async def install_script():
 
 
 @host_agent_bp.get("/agent.py")
+@public_route("fetched by the installer and by agent self-update; holds no secrets")
 async def agent_source():
     try:
         body = AGENT_SOURCE_PATH.read_text(encoding="utf-8")
@@ -345,6 +447,28 @@ async def agent_source():
     return Response(body, mimetype="text/x-python")
 
 
+_FLEET_ORDER = {"stale": 0, "pending": 1, "up": 2}
+
+
+def fleet_state(reg, now: datetime, stale_after: int) -> str:
+    """An agent as the fleet table shows it: "up" while it reports, "stale"
+    once its last report is older than stale_after, "pending" if it never has."""
+    if reg.last_seen_at is None:
+        return "pending"
+    return "stale" if (now - reg.last_seen_at).total_seconds() > stale_after else "up"
+
+
+def fleet_rows(regs, hosts_by_id: dict, now: datetime, stale_after: int) -> list[dict]:
+    """The fleet table: silent agents first, then never-reported, then live,
+    each by host name."""
+    rows = []
+    for r in regs:
+        host = hosts_by_id.get(r.host_id)
+        rows.append({"reg": r, "host": host, "name": host.name if host else r.host_id,
+                     "state": fleet_state(r, now, stale_after)})
+    return sorted(rows, key=lambda x: (_FLEET_ORDER[x["state"]], x["name"].lower()))
+
+
 def _stale_after_seconds() -> int:
     return int(
         current_app.config.get("PLUGINS", {})
@@ -353,124 +477,8 @@ def _stale_after_seconds() -> int:
     )
 
 
-async def _fleet_rows(session) -> list[dict]:
-    """One row per registered host with its latest host-level metrics + stale flag.
-
-    Only bare host-level resources are used here (sub-resources carry a ':' and
-    are skipped) so the fleet glance stays one line per host.
-    """
-    regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
-    host_ids = [r.host_id for r in regs]
-    hosts = {
-        h.id: h for h in (await session.execute(
-            select(Host).where(Host.id.in_(host_ids))
-        )).scalars().all()
-    } if host_ids else {}
-
-    subq = (
-        select(
-            PluginMetric.resource_name,
-            PluginMetric.metric_name,
-            func.max(PluginMetric.recorded_at).label("max_ts"),
-        )
-        .where(PluginMetric.source_module == SOURCE_MODULE)
-        .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
-    ).subquery()
-    latest_rows = (await session.execute(
-        select(PluginMetric).join(
-            subq,
-            (PluginMetric.resource_name == subq.c.resource_name) &
-            (PluginMetric.metric_name == subq.c.metric_name) &
-            (PluginMetric.recorded_at == subq.c.max_ts),
-        ).where(PluginMetric.source_module == SOURCE_MODULE)
-    )).scalars().all()
-
-    latest: dict[str, dict[str, float]] = {}
-    root_disk: dict[str, float] = {}  # host_name → root (/) disk %
-    for row in latest_rows:
-        if row.resource_name.endswith(":/") and row.metric_name == "disk_used_pct":
-            root_disk[row.resource_name[:-2]] = row.value
-        if ":" in row.resource_name:
-            continue
-        latest.setdefault(row.resource_name, {})[row.metric_name] = row.value
-
-    # Per-host recent series for the inline sparklines (cpu/mem/load host-level,
-    # disk from the root mount), so each metric shows its trend beside the value —
-    # the host-panel at-a-glance, on the fleet widget. 1h window keeps it cheap.
-    host_names = [h.name for h in hosts.values()]
-    spark_series: dict[str, dict[str, list]] = {
-        n: {"cpu": [], "mem": [], "disk": [], "load": []} for n in host_names
-    }
-    if host_names:
-        since = datetime.now(timezone.utc) - timedelta(hours=1)
-        sp_rows = (await session.execute(
-            select(PluginMetric).where(
-                PluginMetric.source_module == SOURCE_MODULE,
-                PluginMetric.recorded_at >= since,
-                or_(
-                    and_(PluginMetric.resource_name.in_(host_names),
-                         PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))),
-                    and_(PluginMetric.resource_name.in_([n + ":/" for n in host_names]),
-                         PluginMetric.metric_name == "disk_used_pct"),
-                ),
-            ).order_by(PluginMetric.recorded_at)
-        )).scalars().all()
-        for r in sp_rows:
-            if r.resource_name.endswith(":/"):
-                name = r.resource_name[:-2]
-                if name in spark_series:
-                    spark_series[name]["disk"].append(r.value)
-            else:
-                key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name)
-                if key and r.resource_name in spark_series:
-                    spark_series[r.resource_name][key].append(r.value)
-
-    from steward.core.status import sparkline_svg
-
-    stale_after = _stale_after_seconds()
-    now = datetime.now(timezone.utc)
-    rows = []
-    for reg in regs:
-        host = hosts.get(reg.host_id)
-        if host is None:
-            continue
-        m = latest.get(host.name, {})
-        ls = reg.last_seen_at
-        stale = ls is None or (now - ls).total_seconds() > stale_after
-        ser = spark_series.get(host.name, {})
-        sparks = {
-            k: (sparkline_svg(v[-60:], width=72, height=16) if len(v) >= 2 else "")
-            for k, v in ser.items()
-        }
-        rows.append({
-            "host": host,
-            "reg": reg,
-            "stale": stale,
-            "cpu_pct": m.get("cpu_pct"),
-            "mem_used_pct": m.get("mem_used_pct"),
-            "disk_worst": m.get("disk_used_pct_worst"),  # health dot only
-            "disk_root": root_disk.get(host.name),       # displayed at-a-glance
-            "load_1m": m.get("load_1m"),
-            "temp_max": m.get("temp_c_max"),
-            "net_rx_bps": m.get("net_rx_bps"),
-            "net_tx_bps": m.get("net_tx_bps"),
-            "disk_read_bps": m.get("disk_read_bps"),
-            "disk_write_bps": m.get("disk_write_bps"),
-            "sparks": sparks,
-        })
-    rows.sort(key=lambda r: r["host"].name.lower())
-    return rows
-
-
-@host_agent_bp.get("/widget")
-async def widget_table():
-    """Fleet-glance dashboard widget: one row per monitored host, latest metrics."""
-    async with current_app.db_sessionmaker() as session:
-        rows = await _fleet_rows(session)
-    return await render_template("widget_table.html", rows=rows)
-
-
 @host_agent_bp.get("/widget/history")
+@require_role(UserRole.viewer)
 async def widget_history():
     host_id = request.args.get("host_id", "")
     try:
@@ -664,37 +672,25 @@ async def host_vitals(host_id: str):
             HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
         latest = await _latest_metrics_for_host(db, host.name) if reg else {}
 
-        # Recent series for the at-a-glance sparklines (cpu/mem/load host-level,
-        # disk from the root mount sub-resource).
+        # The last 6 h for the at-a-glance sparklines (cpu/mem/load host-level,
+        # disk from the root mount), averaged into ~120 points in SQL, so the
+        # line spans the whole window and only what is drawn is fetched.
         series_raw: dict[str, list[float]] = {"cpu": [], "mem": [], "disk": [], "load": []}
         if reg:
             since = datetime.now(timezone.utc) - timedelta(hours=6)
-            spark_rows = (await db.execute(
-                select(PluginMetric).where(
-                    PluginMetric.source_module == SOURCE_MODULE,
-                    PluginMetric.recorded_at >= since,
-                    or_(
-                        and_(PluginMetric.resource_name == host.name,
-                             PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))),
-                        and_(PluginMetric.resource_name == host.name + ":/",
-                             PluginMetric.metric_name == "disk_used_pct"),
-                    ),
-                ).order_by(PluginMetric.recorded_at)
-            )).scalars().all()
-            for r in spark_rows:
-                if r.resource_name == host.name:
-                    key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name)
-                    if key:
-                        series_raw[key].append(r.value)
-                else:
-                    series_raw["disk"].append(r.value)
+            for key, resource, metric in (("cpu", host.name, "cpu_pct"),
+                                          ("mem", host.name, "mem_used_pct"),
+                                          ("load", host.name, "load_1m"),
+                                          ("disk", host.name + ":/", "disk_used_pct")):
+                series_raw[key] = [v for _t, v in await metric_points(
+                    db, SOURCE_MODULE, resource, metric, since, target=120)]
 
     from steward.core.status import sparkline_svg
     hostlvl = latest.get(host.name, {})
     # At-a-glance disk = the root filesystem (what people actually care about),
     # not the "worst" mount which is opaque. Worst stays on the full-metrics page.
     disk_root = (latest.get(host.name + ":/", {}) or {}).get("disk_used_pct")
-    sparks = {k: sparkline_svg(v[-120:]) for k, v in series_raw.items()}
+    sparks = {k: sparkline_svg(v, height=24, fill_width=True) for k, v in series_raw.items()}
     # Normalize load by core count so it's comparable across hardware (load/core
     # as %). Cores derived from the per-core CPU sub-resources already collected.
     cores = sum(1 for k in latest if k.startswith(host.name + ":core"))
@@ -708,11 +704,14 @@ async def host_vitals(host_id: str):
     ls = reg.last_seen_at if reg else None
     reporting = bool(reg) and ls is not None
     stale = reporting and (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
+    cpu, mem = hostlvl.get("cpu_pct"), hostlvl.get("mem_used_pct")
+    worst = worst_reading((("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("Disk /", disk_root, "disk"),
+                           ("Load per core", load_per_core, "load")),
+                          current_app.config.get("THRESHOLDS", {}))
     return await render_template(
         "_host_vitals.html",
-        reg=reg, cpu=hostlvl.get("cpu_pct"), mem=hostlvl.get("mem_used_pct"),
-        disk_root=disk_root, load_per_core=load_per_core, sparks=sparks, psi=psi,
-        reporting=reporting, stale=stale,
+        reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core,
+        sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst,
     )
 
 
@@ -784,11 +783,10 @@ async def fleet():
     managed_key_set = bool(
         (current_app.config.get("ANSIBLE", {}).get("ssh_public_key") or "").strip())
 
+    now, stale_after = datetime.now(timezone.utc), _stale_after_seconds()
     return await render_template(
         "settings_list.html",
-        registrations=[
-            {"reg": r, "host": hosts_by_id.get(r.host_id)} for r in regs
-        ],
+        registrations=fleet_rows(regs, hosts_by_id, now, stale_after),
         new_token=new_token,
         install_url=install_url,
         ansible_available=ansible_available,
@@ -798,35 +796,24 @@ async def fleet():
     )
 
 
-@host_agent_bp.post("/fleet/add-host")
+@host_agent_bp.post("/fleet/<host_id>/register")
 @require_role(UserRole.admin)
-async def add_host():
-    form = await request.form
-    name = (form.get("name") or "").strip()
-    address = (form.get("address") or "").strip()
-    if not name:
-        return _error(400, "missing_name")
-
+async def register_host(host_id: str):
+    """Issue an agent token for an existing host, for a manual install: the
+    fleet page then shows the one-line install command, once. Hosts are added
+    on Hosts → Add host; this is only the agent half (#5282)."""
     async with current_app.db_sessionmaker() as session:
-        host = (await session.execute(
-            select(Host).where(Host.name == name))).scalar_one_or_none()
+        host = await session.get(Host, host_id)
         if host is None:
-            host = Host(name=name, address=address)
-            session.add(host)
-            await session.flush()
+            return _error(404, "not_found")
         existing = (await session.execute(
             select(HostAgentRegistration).where(
                 HostAgentRegistration.host_id == host.id))).scalar_one_or_none()
         if existing is not None:
-            await session.rollback()
             return _error(400, "already_registered")
-
         raw, hashed = _new_token_pair()
-        reg = HostAgentRegistration(host_id=host.id, token_hash=hashed)
-        session.add(reg)
+        session.add(HostAgentRegistration(host_id=host.id, token_hash=hashed))
         await session.commit()
-        host_id = host.id
-
     return redirect(url_for("host_agent.fleet") + f"?new_token={raw}&host_id={host_id}")
 
 
@@ -947,6 +934,7 @@ async def deploy_via_ansible():
         playbook_path="host_agent/install.yml",
         inventory_content=inventory_content,
         inventory_scope=scope,
+        redact=list(tokens.values()),
         # extra-vars outrank the playbook's `vars:` defaults; token stays per-host.
         params={"extra_vars_map": {"steward_url": url, "agent_interval": interval}},
         triggered_by=session.get("user_id"),
@@ -1024,6 +1012,7 @@ async def provision_via_ansible():
         playbook_path="host_agent/provision.yml",
         inventory_content=inventory_content,
         inventory_scope=scope,
+        redact=list(tokens.values()),
         # extra_vars_map → a JSON -e @file: highest precedence (beats play vars)
         # and space-safe (steward_pubkey carries a comment with a space).
         params={"extra_vars_map": {

plugins/host_agent/scheduler.py

@@ -1,11 +1,12 @@
 # plugins/host_agent/scheduler.py
 from __future__ import annotations
 import logging
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timezone
 from typing import Iterable
 
 from sqlalchemy import select
 
+from steward.core.alerts import MetricBatch
 from steward.core.scheduler import ScheduledTask
 from steward.models.hosts import Host
 from .models import HostAgentRegistration
@@ -13,66 +14,52 @@ from .models import HostAgentRegistration
 logger = logging.getLogger(__name__)
 
 
-def _filter_stale(
-    regs: Iterable,
-    *,
-    now: datetime,
-    stale_after_seconds: int,
-) -> list:
-    """Pure staleness filter: returns the subset with last_seen_at strictly
-    older than (now - stale_after_seconds). Rows with last_seen_at=None are
-    never stale (they are unregistered-in-practice)."""
-    cutoff = now - timedelta(seconds=stale_after_seconds)
-    return [
-        r for r in regs
-        if r.last_seen_at is not None and r.last_seen_at < cutoff
-    ]
+def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
+                       now: datetime) -> list[tuple[str, float]]:
+    """(host name, seconds since its agent last reported) per registration.
+
+    An agent that has never reported counts from when it was registered. A
+    registration whose host is gone is skipped.
+    """
+    out = []
+    for r in regs:
+        name = host_names.get(r.host_id)
+        seen = r.last_seen_at or r.created_at
+        if name is None or seen is None:
+            continue
+        out.append((name, max(0.0, (now - seen).total_seconds())))
+    return out
 
 
-async def find_stale_registrations(app, stale_after_seconds: int = 180) -> list[dict]:
+async def record_seconds_since_seen(app) -> None:
+    """Record host_agent/<host>/seconds_since_seen for every registered agent,
+    so "agent offline" is an ordinary alert rule (seconds_since_seen > 180)."""
+    now = datetime.now(timezone.utc)
     async with app.db_sessionmaker() as session:
-        all_regs = (await session.execute(
-            select(HostAgentRegistration)
-        )).scalars().all()
-        stale_rows = _filter_stale(
-            all_regs,
-            now=datetime.now(timezone.utc),
-            stale_after_seconds=stale_after_seconds,
-        )
-        if not stale_rows:
-            return []
-        hosts = {
-            h.id: h for h in (await session.execute(
-                select(Host).where(Host.id.in_([r.host_id for r in stale_rows]))
-            )).scalars().all()
-        }
-    return [
-        {
-            "host_id": r.host_id,
-            "host_name": hosts[r.host_id].name if r.host_id in hosts else "?",
-            "last_seen_at": r.last_seen_at,
-        }
-        for r in stale_rows
-    ]
+        async with session.begin():
+            regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
+            if not regs:
+                return
+            host_names = dict((await session.execute(
+                select(Host.id, Host.name).where(Host.id.in_([r.host_id for r in regs]))
+            )).all())
+            batch = MetricBatch()
+            for name, seconds in seconds_since_seen(regs, host_names, now):
+                batch.add(source_module="host_agent", resource_name=name,
+                          metric_name="seconds_since_seen", value=seconds, recorded_at=now)
+            await batch.record(session)
 
 
 def make_task(app) -> ScheduledTask:
-    async def _check_stale():
+    async def _record():
         try:
-            stale = await find_stale_registrations(app)
+            await record_seconds_since_seen(app)
         except Exception:
-            logger.exception("host_agent stale check failed")
-            return
-        if stale:
-            logger.info(
-                "host_agent: %d stale agent(s): %s",
-                len(stale),
-                [s["host_name"] for s in stale],
-            )
+            logger.exception("host_agent: recording seconds_since_seen failed")
 
     return ScheduledTask(
-        name="host_agent_stale_check",
-        coro_factory=_check_stale,
+        name="host_agent_seconds_since_seen",
+        coro_factory=_record,
         interval_seconds=60,
         run_on_startup=False,
     )

plugins/host_agent/templates/_host_metrics.html

@@ -1,32 +1,15 @@
 {# Current-state fragment for the full-metrics page — polled live via HTMX.
-   Self-contained (defines its own format macros) so each poll re-renders the
-   latest snapshot the server has. #}
-{% macro fmt_bps(v) %}
-{%- if v is none -%}—
-{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s
-{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s
-{%- else -%}{{ "%.0f"|format(v) }} B/s
-{%- endif -%}
-{% endmacro %}
-{% macro fmt_bytes(v) %}
-{%- if v is none -%}—
-{%- elif v >= 1125899906842624 -%}{{ "%.1f"|format(v / 1125899906842624) }} PB
-{%- elif v >= 1099511627776 -%}{{ "%.1f"|format(v / 1099511627776) }} TB
-{%- elif v >= 1073741824 -%}{{ "%.1f"|format(v / 1073741824) }} GB
-{%- elif v >= 1048576 -%}{{ "%.0f"|format(v / 1048576) }} MB
-{%- else -%}{{ "%.0f"|format(v / 1024) }} KB
-{%- endif -%}
-{% endmacro %}
-{% macro bar(pct) %}
-{%- set p = pct if pct is not none else 0 -%}
-<div style="height:8px;background:var(--bg-elevated);border-radius:4px;overflow:hidden;border:1px solid var(--border);">
-  <div style="height:100%;width:{{ p|round(1) }}%;background:{% if pct is none %}var(--text-dim){% elif pct < 70 %}var(--green){% elif pct < 90 %}var(--yellow){% else %}var(--red){% endif %};"></div>
-</div>
+   Each poll re-renders the latest snapshot the server has. #}
+{% from "_macros.html" import status_badge %}
+{# A percentage bar graded by Settings → Thresholds for `kind` (cpu, mem, disk). #}
+{% macro bar(pct, kind) %}
+<div class="meter"><div class="meter-fill meter-{{ status_class(pct, kind) }}" style="width:{{ (pct or 0)|round(1) }}%;"></div></div>
 {% endmacro %}
+{% macro gauge(value, kind, text) %}<span class="stat-val tone-{{ status_class(value, kind) }}" style="font-size:1.6rem;">{{ text }}</span>{% endmacro %}
 
 {# ── Identity / metadata ─────────────────────────────────────────────────── #}
 <div class="card" style="display:flex;flex-wrap:wrap;gap:0.5rem 1.25rem;align-items:center;font-size:0.82rem;color:var(--text-muted);">
-  <span>{% if stale %}<span class="badge badge-red">stale</span>{% else %}<span class="badge badge-green">live</span>{% endif %}</span>
+  <span>{{ status_badge('stale' if stale else 'up', 'Stale' if stale else 'Live') }}</span>
   <span><span style="color:var(--text-dim);">Address</span> {{ host.address or "—" }}</span>
   {% if reg %}
   {% if reg.distro %}<span><span style="color:var(--text-dim);">OS</span> {{ reg.distro }}</span>{% endif %}
@@ -34,8 +17,8 @@
   {% if reg.arch %}<span><span style="color:var(--text-dim);">Arch</span> {{ reg.arch }}</span>{% endif %}
   {% if reg.agent_version %}<span><span style="color:var(--text-dim);">Agent</span> v{{ reg.agent_version }}</span>{% endif %}
   {% set up = hostlvl.get('uptime_secs') %}
-  {% if up %}<span><span style="color:var(--text-dim);">Uptime</span> {{ (up // 86400)|int }}d {{ ((up % 86400) // 3600)|int }}h</span>{% endif %}
-  <span><span style="color:var(--text-dim);">Last seen</span> {{ reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") ~ " UTC" if reg.last_seen_at else "never" }}</span>
+  {% if up %}<span><span style="color:var(--text-dim);">Uptime</span> {{ up|duration }}</span>{% endif %}
+  <span><span style="color:var(--text-dim);">Last seen</span> {{ reg.last_seen_at|ago if reg.last_seen_at else "never" }}</span>
   {% else %}
   <span>No agent registration found for this host.</span>
   {% endif %}
@@ -50,16 +33,16 @@
   <div class="card" style="margin-bottom:0;">
     <div class="section-title" style="margin-bottom:0.35rem;">CPU</div>
     {% set c = hostlvl.get('cpu_pct') %}
-    <span class="stat-val" style="font-size:1.6rem;color:{% if c is none %}var(--text-dim){% elif c < 70 %}var(--green){% elif c < 90 %}var(--yellow){% else %}var(--red){% endif %};">{% if c is not none %}{{ "%.0f"|format(c) }}%{% else %}—{% endif %}</span>
-    <div style="margin-top:0.5rem;">{{ bar(c) }}</div>
+    {{ gauge(c, 'cpu', ("%.0f%%"|format(c)) if c is not none else "—") }}
+    <div style="margin-top:0.5rem;">{{ bar(c, 'cpu') }}</div>
   </div>
   <div class="card" style="margin-bottom:0;">
     <div class="section-title" style="margin-bottom:0.35rem;">Memory</div>
     {% set mp = hostlvl.get('mem_used_pct') %}
-    <span class="stat-val" style="font-size:1.6rem;color:{% if mp is none %}var(--text-dim){% elif mp < 70 %}var(--green){% elif mp < 90 %}var(--yellow){% else %}var(--red){% endif %};">{% if mp is not none %}{{ "%.0f"|format(mp) }}%{% else %}—{% endif %}</span>
-    <div style="margin-top:0.5rem;">{{ bar(mp) }}</div>
+    {{ gauge(mp, 'mem', ("%.0f%%"|format(mp)) if mp is not none else "—") }}
+    <div style="margin-top:0.5rem;">{{ bar(mp, 'mem') }}</div>
     <div style="margin-top:0.45rem;font-size:0.72rem;color:var(--text-dim);">
-      avail {{ fmt_bytes(hostlvl.get('mem_available_bytes')) }} · cache {{ fmt_bytes(hostlvl.get('mem_cached_bytes')) }} · swap {{ fmt_bytes(hostlvl.get('swap_used_bytes')) }}
+      avail {{ hostlvl.get('mem_available_bytes')|bytes }} · cache {{ hostlvl.get('mem_cached_bytes')|bytes }} · swap {{ hostlvl.get('swap_used_bytes')|bytes }}
     </div>
   </div>
   <div class="card" style="margin-bottom:0;">
@@ -73,22 +56,22 @@
   <div class="card" style="margin-bottom:0;">
     <div class="section-title" style="margin-bottom:0.35rem;">Network</div>
     <div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
-      <div><span style="color:var(--green);">↓</span> {{ fmt_bps(hostlvl.get('net_rx_bps')) }}</div>
-      <div><span style="color:var(--red);">↑</span> {{ fmt_bps(hostlvl.get('net_tx_bps')) }}</div>
+      <div><span style="color:var(--green);">↓</span> {{ hostlvl.get('net_rx_bps')|rate }}</div>
+      <div><span style="color:var(--red);">↑</span> {{ hostlvl.get('net_tx_bps')|rate }}</div>
     </div>
   </div>
   <div class="card" style="margin-bottom:0;">
     <div class="section-title" style="margin-bottom:0.35rem;">Disk I/O</div>
     <div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
-      <div><span style="color:var(--text-dim);">rd</span> {{ fmt_bps(hostlvl.get('disk_read_bps')) }}</div>
-      <div><span style="color:var(--text-dim);">wr</span> {{ fmt_bps(hostlvl.get('disk_write_bps')) }}</div>
+      <div><span style="color:var(--text-dim);">rd</span> {{ hostlvl.get('disk_read_bps')|rate }}</div>
+      <div><span style="color:var(--text-dim);">wr</span> {{ hostlvl.get('disk_write_bps')|rate }}</div>
     </div>
   </div>
   {% if hostlvl.get('temp_c_max') is not none %}
   <div class="card" style="margin-bottom:0;">
     <div class="section-title" style="margin-bottom:0.35rem;">Temp (max)</div>
     {% set t = hostlvl.get('temp_c_max') %}
-    <span class="stat-val" style="font-size:1.6rem;color:{% if t < 70 %}var(--green){% elif t < 85 %}var(--yellow){% else %}var(--red){% endif %};">{{ "%.0f"|format(t) }}°C</span>
+    {{ gauge(t, 'temp', "%.0f°C"|format(t)) }}
   </div>
   {% endif %}
   {% if hostlvl.get('psi_mem_some_avg10') is not none %}
@@ -113,7 +96,7 @@
       <div style="display:flex;justify-content:space-between;font-size:0.72rem;color:var(--text-dim);margin-bottom:0.2rem;">
         <span>core {{ idx }}</span><span>{% if pct is not none %}{{ "%.0f"|format(pct) }}%{% else %}—{% endif %}</span>
       </div>
-      {{ bar(pct) }}
+      {{ bar(pct, 'cpu') }}
     </div>
     {% endfor %}
   </div>
@@ -128,9 +111,9 @@
   <div style="margin-bottom:0.6rem;">
     <div style="display:flex;justify-content:space-between;font-size:0.8rem;margin-bottom:0.2rem;">
       <span style="font-family:ui-monospace,monospace;">{{ mount }}</span>
-      <span style="color:var(--text-muted);">{{ fmt_bytes(m.get('disk_used_bytes')) }} / {{ fmt_bytes(m.get('disk_total_bytes')) }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %}</span>
+      <span style="color:var(--text-muted);">{{ m.get('disk_used_bytes')|bytes }} / {{ m.get('disk_total_bytes')|bytes }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %}</span>
     </div>
-    {{ bar(m.get('disk_used_pct')) }}
+    {{ bar(m.get('disk_used_pct'), 'disk') }}
   </div>
   {% endfor %}
 </div>
@@ -145,7 +128,7 @@
     {% for iface, m in nets.items() %}
     <div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
       <span style="font-family:ui-monospace,monospace;">{{ iface }}</span>
-      <span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">↓ {{ fmt_bps(m.get('net_rx_bps')) }} · ↑ {{ fmt_bps(m.get('net_tx_bps')) }}</span>
+      <span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">↓ {{ m.get('net_rx_bps')|rate }} · ↑ {{ m.get('net_tx_bps')|rate }}</span>
     </div>
     {% endfor %}
   </div>
@@ -156,7 +139,7 @@
     {% for dev, m in disks_io.items() %}
     <div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
       <span style="font-family:ui-monospace,monospace;">{{ dev }}</span>
-      <span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">rd {{ fmt_bps(m.get('disk_read_bps')) }} · wr {{ fmt_bps(m.get('disk_write_bps')) }}</span>
+      <span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">rd {{ m.get('disk_read_bps')|rate }} · wr {{ m.get('disk_write_bps')|rate }}</span>
     </div>
     {% endfor %}
   </div>
@@ -172,7 +155,7 @@
     {% for label, c in temps.items() %}
     <div style="display:flex;justify-content:space-between;gap:0.5rem;font-size:0.8rem;padding:0.15rem 0;">
       <span style="white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ label }}</span>
-      <span style="color:{% if c is none %}var(--text-dim){% elif c < 70 %}var(--green){% elif c < 85 %}var(--yellow){% else %}var(--red){% endif %};font-variant-numeric:tabular-nums;">{% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %}</span>
+      <span class="tone-{{ status_class(c, 'temp') }}" style="font-variant-numeric:tabular-nums;">{% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %}</span>
     </div>
     {% endfor %}
   </div>

plugins/host_agent/templates/_host_vitals.html

@@ -1,35 +1,33 @@
-{# Host vitals strip — compact CPU/MEM/DISK/LOAD + pressure, live-polled into the
-   host hub. Renders nothing until the agent reports (the Agent panel below then
-   shows provisioning). #}
+{# Host health header — live polled into the host hub: live or stale, the worst
+   reading against Settings → Thresholds, last seen, then CPU / memory / disk /
+   load tiles graded by the same cutoffs. Renders nothing until the agent
+   reports (the Agent panel below then shows provisioning). #}
+{% from "_macros.html" import status_badge, stat_tile %}
+{% macro pct(v) %}{{ '%.0f%%'|format(v) if v is not none else '—' }}{% endmacro %}
 {% if reporting %}
-{% set lbl = "font-size:0.68rem;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em;" %}
-{% macro vital(label, value, text, kind, spark) %}
-<div style="min-width:88px;">
-  <div style="{{ lbl }}">{{ label }}</div>
-  <div style="font-size:1.5rem;font-weight:700;line-height:1.1;{{ threshold_style(value, kind) }}">{{ text }}</div>
-  <div style="margin-top:0.2rem;">{{ spark | safe }}</div>
-</div>
-{% endmacro %}
-<div class="card" style="display:flex;flex-wrap:wrap;align-items:flex-start;gap:1rem 2rem;margin-bottom:1rem;">
-  {{ vital("CPU", cpu, ('%.0f%%'|format(cpu) if cpu is not none else '—'), 'cpu', sparks.cpu) }}
-  {{ vital("Memory", mem, ('%.0f%%'|format(mem) if mem is not none else '—'), 'mem', sparks.mem) }}
-  {{ vital("Disk /", disk_root, ('%.0f%%'|format(disk_root) if disk_root is not none else '—'), 'disk', sparks.disk) }}
-  {{ vital("Load /core", load_per_core, ('%d%%'|format(load_per_core) if load_per_core is not none else '—'), 'load', sparks.load) }}
-
-  <div style="margin-left:auto;display:flex;flex-direction:column;align-items:flex-end;gap:0.3rem;font-size:0.75rem;color:var(--text-muted);text-align:right;">
-    <span>
-      {% if stale %}<span class="badge badge-red">stale</span>{% else %}<span class="badge badge-green">live</span>{% endif %}
-      {% if reg and reg.agent_version %}<span style="margin-left:0.3rem;">v{{ reg.agent_version }}</span>{% endif %}
+<section class="card host-health" aria-label="Host health">
+  <div class="host-health-head">
+    {{ status_badge('stale' if stale else 'up', 'Stale' if stale else 'Live') }}
+    <span class="host-health-summary">
+      {%- if stale %}No report since {{ reg.last_seen_at|ago }}; the readings below are from then.
+      {%- elif worst %}<span class="tone-{{ worst[2] }}">{{ worst[0] }} {{ pct(worst[1]) }}</span> is past its {{ "critical" if worst[2] == "crit" else "warning" }} threshold.
+      {%- else %}Every reading is within its thresholds.{% endif -%}
     </span>
-    {% if psi.cpu is not none or psi.mem is not none or psi.io is not none %}
-    <span title="Pressure stall — % of the last 10s tasks waited for the resource">
-      <span style="color:var(--text-dim);text-transform:uppercase;font-size:0.66rem;letter-spacing:0.04em;">Pressure 10s</span>
-      cpu {{ '%.0f%%'|format(psi.cpu) if psi.cpu is not none else '—' }}
-      · mem {{ '%.0f%%'|format(psi.mem) if psi.mem is not none else '—' }}
-      · io {{ '%.0f%%'|format(psi.io) if psi.io is not none else '—' }}
+    <span class="problem-detail">
+      {%- if not stale %}seen {{ reg.last_seen_at|ago }}{% endif %}
+      {%- if reg.agent_version %}{{ " · " if not stale }}agent v{{ reg.agent_version }}{% endif -%}
     </span>
-    {% endif %}
-    {% if reg and reg.last_seen_at %}<span>seen {{ reg.last_seen_at.strftime("%H:%M:%S") }} UTC</span>{% endif %}
   </div>
-</div>
+  <div class="stat-strip host-health-tiles" style="margin-bottom:0;">
+    {{ stat_tile("CPU", pct(cpu), state=status_class(cpu, 'cpu'), hint=sparks.cpu|safe) }}
+    {{ stat_tile("Memory", pct(mem), state=status_class(mem, 'mem'), hint=sparks.mem|safe) }}
+    {{ stat_tile("Disk /", pct(disk_root), state=status_class(disk_root, 'disk'), hint=sparks.disk|safe) }}
+    {{ stat_tile("Load per core", pct(load_per_core), state=status_class(load_per_core, 'load'), hint=sparks.load|safe) }}
+  </div>
+  {% if psi.cpu is not none or psi.mem is not none or psi.io is not none %}
+  <p class="field-hint" style="margin-top:0.6rem;" title="Pressure stall: the share of the last 10 s that tasks waited for the resource">
+    Pressure (10 s): cpu {{ pct(psi.cpu) }} · mem {{ pct(psi.mem) }} · io {{ pct(psi.io) }}
+  </p>
+  {% endif %}
+</section>
 {% endif %}

plugins/host_agent/templates/host_detail.html

@@ -54,30 +54,30 @@
     interaction: { mode: "index", intersect: false },
     elements: { point: { radius: 0 } },
     scales: {
-      x: { type: "linear", ticks: { callback: (v) => fmtTime(v), maxTicksLimit: 8, color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
-      y: { beginAtZero: true, max: ymax, ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
+      x: { type: "linear", ticks: { callback: (v) => fmtTime(v), maxTicksLimit: 8, font: { size: 10 } } },
+      y: { beginAtZero: true, max: ymax, ticks: { font: { size: 10 } } },
     },
     plugins: {
-      legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } },
+      legend: { labels: { boxWidth: 10, font: { size: 11 } } },
       tooltip: { callbacks: { title: (items) => items.length ? fmtTime(items[0].parsed.x) : "" } },
     },
   });
   const defs = {
     "chart-util": { ymax: 100, ds: [
-      { key: "cpu_pct", label: "CPU %", color: "#c8a840" },
-      { key: "mem_used_pct", label: "Mem %", color: "#4aa86a" },
-      { key: "disk_used_pct_worst", label: "Disk % (worst)", color: "#c87840" },
+      { key: "cpu_pct", label: "CPU %", color: stewardColor("--gold") },
+      { key: "mem_used_pct", label: "Mem %", color: stewardColor("--green") },
+      { key: "disk_used_pct_worst", label: "Disk % (worst)", color: stewardColor("--yellow") },
     ] },
     "chart-net": { ymax: undefined, ds: [
-      { key: "net_rx_bps", label: "Net RX", color: "#4aa86a" },
-      { key: "net_tx_bps", label: "Net TX", color: "#c84048" },
-      { key: "disk_read_bps", label: "Disk read", color: "#6aa0d0" },
-      { key: "disk_write_bps", label: "Disk write", color: "#c8a840" },
+      { key: "net_rx_bps", label: "Net RX", color: stewardColor("--green") },
+      { key: "net_tx_bps", label: "Net TX", color: stewardColor("--red") },
+      { key: "disk_read_bps", label: "Disk read", color: stewardColor("--blue") },
+      { key: "disk_write_bps", label: "Disk write", color: stewardColor("--gold") },
     ] },
     "chart-pressure": { ymax: undefined, ds: [
-      { key: "load_1m", label: "Load 1m", color: "#c8a840" },
-      { key: "psi_mem_some_avg10", label: "Mem PSI %", color: "#c84048" },
-      { key: "temp_c_max", label: "Temp °C", color: "#c87840" },
+      { key: "load_1m", label: "Load 1m", color: stewardColor("--gold") },
+      { key: "psi_mem_some_avg10", label: "Mem PSI %", color: stewardColor("--red") },
+      { key: "temp_c_max", label: "Temp °C", color: stewardColor("--yellow") },
     ] },
   };
   const charts = {};

plugins/host_agent/templates/install.sh.j2

@@ -45,6 +45,10 @@ cat > "$CONF_FILE" <<EOF
 url = $STEWARD_URL
 token = $AGENT_TOKEN
 interval_seconds = 30
+# Container logs are collected by default. To opt this host out entirely:
+#   docker_logs_enabled = false
+# To skip specific noisy containers (comma-separated names):
+#   docker_log_exclude = watchtower, some-chatty-service
 EOF
 chown "root:$AGENT_USER" "$CONF_FILE"
 chmod 0640 "$CONF_FILE"

plugins/host_agent/templates/panel.html

@@ -4,9 +4,9 @@
   <div style="display:flex;align-items:baseline;justify-content:space-between;margin-bottom:0.75rem;gap:1rem;flex-wrap:wrap;">
     <h3 class="section-title" style="margin-bottom:0;">Agent</h3>
     {% if reporting %}
-    <span style="font-size:0.78rem;color:{{ 'var(--yellow)' if stale else 'var(--green)' }};">
+    <span style="font-size:0.78rem;" class="tone-{{ 'warn' if stale else 'ok' }}">
       {{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %}
-      · last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }}
+      · last seen {{ reg.last_seen_at|ago }}
     </span>
     {% elif reg %}
     <span style="font-size:0.78rem;color:var(--text-muted);">pending — no check-in yet</span>
@@ -15,52 +15,57 @@
 
   {% if reporting %}
   {# Reporting: live vitals are shown by the vitals strip above; this panel is
-     lifecycle/management only. #}
-  <div style="display:flex;gap:0.5rem;flex-wrap:wrap;align-items:center;">
-    <a href="/plugins/host_agent/{{ host.id }}/" class="btn btn-sm btn-ghost">Full metrics →</a>
-    {% if session.user_role == 'admin' %}
-    {% if ansible_available and target %}
-    <form method="post" action="/plugins/host_agent/update" style="margin:0;">
-      <input type="hidden" name="inventory_scope" value="{{ scope }}">
-      <button type="submit" class="btn btn-sm" title="Refresh agent.py + restart (token preserved)">Update agent</button>
-    </form>
-    {% endif %}
-    <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/rotate-token" style="margin:0;"
-          onsubmit="return confirm('Rotate token? The agent stops reporting until reinstalled/updated with the new token.');">
-      <button type="submit" class="btn btn-sm btn-ghost">Rotate token</button>
-    </form>
-    <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0;"
-          onsubmit="return confirm('Remove this agent registration? Metrics stop until re-registered.');">
-      <button type="submit" class="btn btn-sm btn-danger">Remove</button>
-    </form>
-    {% endif %}
-  </div>
-  {% if ansible_available and not target %}
-  <p style="color:var(--text-dim);font-size:0.8rem;margin:0.6rem 0 0;">
-    Link an Ansible target (above) to enable one-click agent updates.
-  </p>
-  {% endif %}
+     the way to the full metrics, with the lifecycle actions folded under
+     "Manage agent" (#5278). #}
+  <a href="/plugins/host_agent/{{ host.id }}/" class="btn btn-sm btn-ghost">Full metrics →</a>
 
-  {% if session.user_role == 'admin' and ansible_available and target and managed_key_set %}
+  {% if session.user_role == 'admin' %}
   <details style="margin-top:0.7rem;">
-    <summary style="cursor:pointer;font-size:0.8rem;color:var(--text-muted);">Re-provision (reinstall the steward account + managed key, then the agent)</summary>
-    <p style="font-size:0.78rem;color:var(--text-dim);margin:0.5rem 0;">
-      Use after regenerating the managed key, or if SSH auth as <code>steward</code> breaks.
-      Connects over a one-time bootstrap user + password (not stored).
+    <summary style="cursor:pointer;font-size:0.82rem;color:var(--text-muted);">Manage agent</summary>
+    <div style="display:flex;gap:0.5rem;flex-wrap:wrap;align-items:center;margin-top:0.6rem;">
+      {% if ansible_available and target %}
+      <form method="post" action="/plugins/host_agent/update" style="margin:0;">{{ csrf_field() }}
+        <input type="hidden" name="inventory_scope" value="{{ scope }}">
+        <button type="submit" class="btn btn-sm" title="Refresh agent.py and restart it; the token is kept">Update agent</button>
+      </form>
+      {% endif %}
+      <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/rotate-token" style="margin:0;"
+            data-confirm="Rotate token? The agent stops reporting until reinstalled/updated with the new token.">{{ csrf_field() }}
+        <button type="submit" class="btn btn-sm btn-ghost">Rotate token</button>
+      </form>
+      <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0;"
+            data-confirm="Unregister this host's agent? Its token stops working and metrics stop until it is registered again. The agent itself stays installed on the host.">{{ csrf_field() }}
+        <button type="submit" class="btn btn-sm btn-danger">Unregister agent</button>
+      </form>
+    </div>
+    {% if ansible_available and not target %}
+    <p style="color:var(--text-dim);font-size:0.8rem;margin:0.6rem 0 0;">
+      Link an Ansible target (below) to update the agent from here.
     </p>
-    <form method="post" action="/plugins/host_agent/provision"
-          style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;">
-      <input type="hidden" name="inventory_scope" value="{{ scope }}">
-      <div class="form-group" style="margin:0;">
-        <label>Bootstrap user</label>
-        <input type="text" name="bootstrap_user" required placeholder="root" style="width:8rem;" autocomplete="off">
-      </div>
-      <div class="form-group" style="margin:0;">
-        <label>Bootstrap password</label>
-        <input type="password" name="bootstrap_password" required style="width:10rem;" autocomplete="new-password">
-      </div>
-      <button type="submit" class="btn btn-sm">Re-provision</button>
-    </form>
+    {% endif %}
+
+    {% if ansible_available and target and managed_key_set %}
+    <details style="margin-top:0.7rem;">
+      <summary style="cursor:pointer;font-size:0.8rem;color:var(--text-muted);">Re-provision (reinstall the steward account + managed key, then the agent)</summary>
+      <p style="font-size:0.78rem;color:var(--text-dim);margin:0.5rem 0;">
+        Use after regenerating the managed key, or if SSH auth as <code>steward</code> breaks.
+        Connects over a one-time bootstrap user + password (not stored).
+      </p>
+      <form method="post" action="/plugins/host_agent/provision"
+            style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
+        <input type="hidden" name="inventory_scope" value="{{ scope }}">
+        <div class="form-group" style="margin:0;">
+          <label>Bootstrap user</label>
+          <input type="text" name="bootstrap_user" required placeholder="root" style="width:8rem;" autocomplete="off">
+        </div>
+        <div class="form-group" style="margin:0;">
+          <label>Bootstrap password</label>
+          <input type="password" name="bootstrap_password" required style="width:10rem;" autocomplete="new-password">
+        </div>
+        <button type="submit" class="btn btn-sm">Re-provision</button>
+      </form>
+    </details>
+    {% endif %}
   </details>
   {% endif %}
 
@@ -86,8 +91,7 @@
 
   {% elif not ansible_available %}
   <p style="color:var(--text-dim);font-size:0.85rem;">
-    Ansible isn't available, so the agent can't be deployed from here. Install it manually with the
-    <a href="/plugins/host_agent/fleet/">curl install command</a>.
+    Ansible isn't available, so the agent can't be deployed from here. Install it manually instead (below).
   </p>
 
   {% elif not managed_key_set %}
@@ -96,7 +100,7 @@
               border-radius:6px;padding:0.7rem 0.9rem;display:flex;align-items:center;gap:0.8rem;flex-wrap:wrap;">
     <span style="color:var(--yellow);">⚠</span>
     <span style="font-size:0.84rem;flex:1;min-width:13rem;">No managed SSH key yet — Steward needs one to log into hosts.</span>
-    <form method="post" action="/settings/ansible/generate-key" style="margin:0;">
+    <form method="post" action="/settings/ansible/generate-key" style="margin:0;">{{ csrf_field() }}
       <input type="hidden" name="next" value="/hosts/{{ host.id }}">
       <button type="submit" class="btn btn-sm">Generate managed key</button>
     </form>
@@ -115,7 +119,7 @@
   </p>
   {# Provision: brand-new host (creates steward account + key over a one-time password) #}
   <form method="post" action="/plugins/host_agent/provision"
-        style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;margin-bottom:0.75rem;">
+        style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;margin-bottom:0.75rem;">{{ csrf_field() }}
     <input type="hidden" name="inventory_scope" value="{{ scope }}">
     <div style="font-size:0.78rem;color:var(--text-muted);width:100%;">Provision (first contact — fresh host):</div>
     <div class="form-group" style="margin:0;">
@@ -129,16 +133,26 @@
     <button type="submit" class="btn btn-sm">Provision</button>
   </form>
   {# Install: host already has the steward account (managed key works) #}
-  <form method="post" action="/plugins/host_agent/deploy" style="display:flex;gap:0.6rem;align-items:center;">
+  <form method="post" action="/plugins/host_agent/deploy" style="display:flex;gap:0.6rem;align-items:center;">{{ csrf_field() }}
     <input type="hidden" name="inventory_scope" value="{{ scope }}">
     <span style="font-size:0.78rem;color:var(--text-muted);">Already provisioned?</span>
     <button type="submit" class="btn btn-sm btn-ghost">Install agent (managed key)</button>
   </form>
   {% endif %}
 
+  {% if not reg and session.user_role == 'admin' %}
+  {# A manual install: no Ansible, no SSH. Issues this host's token and shows
+     the one-line install command on the fleet page, once (#5282). #}
+  <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/register"
+        style="display:flex;gap:0.6rem;align-items:center;flex-wrap:wrap;margin:0.75rem 0 0;">{{ csrf_field() }}
+    <span style="font-size:0.78rem;color:var(--text-muted);">Or by hand on the host:</span>
+    <button type="submit" class="btn btn-sm btn-ghost">Install manually</button>
+  </form>
+  {% endif %}
+
   {% if reg and session.user_role == 'admin' %}
   <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0.75rem 0 0;"
-        onsubmit="return confirm('Clear this pending registration? Its token is discarded.');">
+        data-confirm="Clear this pending registration? Its token is discarded.">{{ csrf_field() }}
     <button type="submit" class="btn btn-sm btn-ghost">Clear pending registration</button>
   </form>
   {% endif %}

plugins/host_agent/templates/settings_list.html

@@ -1,5 +1,5 @@
 {% extends "base.html" %}
-{% from "_macros.html" import crumbs %}
+{% from "_macros.html" import crumbs, status_badge %}
 {% block title %}Agent fleet — Steward{% endblock %}
 {% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), ("Agent fleet", "")]) }}{% endblock %}
 {% block content %}
@@ -8,11 +8,12 @@
 </div>
 <p style="color:var(--text-muted);font-size:0.84rem;margin:-0.4rem 0 1.25rem;">
   Bulk agent operations across your inventory. For a single host, use its
-  <a href="/hosts/">host page</a> — provisioning and metrics live there.
+  <a href="/hosts/">host page</a>: provisioning, a manual install command and
+  metrics live there. New hosts are added with <a href="/hosts/?add=1">Add host</a>.
 </p>
 
 {% if new_token %}
-<div class="card" style="background:var(--accent-bg);">
+<div class="card" style="background:var(--fs-accent-faint);">
   <h3>New token — copy this install command now</h3>
   <p style="font-size:0.82rem;color:var(--text-muted);">
     This is the only time the raw token is shown. Rotate if you lose it.
@@ -21,19 +22,43 @@
 </div>
 {% endif %}
 
-<div class="card">
-  <form method="post" action="/plugins/host_agent/fleet/add-host"
-        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
-    <div class="form-group" style="margin-bottom:0;">
-      <label>Host name</label>
-      <input type="text" name="name" required>
-    </div>
-    <div class="form-group" style="margin-bottom:0;">
-      <label>Address (optional)</label>
-      <input type="text" name="address">
-    </div>
-    <button type="submit" class="btn">Add host</button>
-  </form>
+<div class="card-flush" style="margin-bottom:1.5rem;">
+  <div class="table-wrap">
+  <table class="table">
+    <thead>
+      <tr>
+        <th>Status</th><th>Host</th><th>Last seen</th><th>Agent version</th><th>Reported IP</th><th>Distro</th>
+        <th class="td-actions">Actions</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for item in registrations %}
+      <tr>
+        <td>{{ status_badge(item.state, {'up': 'Reporting', 'stale': 'Silent', 'pending': 'Never reported'}[item.state]) }}</td>
+        <td>{% if item.host %}<a href="/hosts/{{ item.host.id }}">{{ item.host.name }}</a>{% else %}<span class="mono">{{ item.reg.host_id }}</span>{% endif %}</td>
+        <td>{% if item.reg.last_seen_at %}<span title="{{ item.reg.last_seen_at.strftime('%Y-%m-%d %H:%M:%S UTC') }}">{{ item.reg.last_seen_at|ago }}</span>{% else %}never{% endif %}</td>
+        <td>{{ item.reg.agent_version or "—" }}</td>
+        <td class="mono">{{ item.reg.host_ip or "—" }}</td>
+        <td>{{ item.reg.distro or "—" }}</td>
+        <td class="td-actions">
+          <form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/rotate-token"
+                style="display:inline;"
+                data-confirm="Rotate the token for {{ item.name }}? The existing agent will stop working until reinstalled with the new token.">{{ csrf_field() }}
+            <button type="submit" class="btn btn-ghost btn-sm">Rotate token</button>
+          </form>
+          <form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/delete"
+                style="display:inline;"
+                data-confirm="Unregister the agent for {{ item.name }}? Its token stops working and metrics stop until it is registered again. The agent itself stays installed on the host.">{{ csrf_field() }}
+            <button type="submit" class="btn btn-danger btn-sm">Unregister</button>
+          </form>
+        </td>
+      </tr>
+      {% else %}
+      <tr><td colspan="7" class="empty">No agents registered yet. <a href="/hosts/?add=1">Add a host</a>, then install its agent from the host's page.</td></tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  </div>
 </div>
 
 {% if ansible_available %}
@@ -63,7 +88,7 @@
     <span style="font-size:0.84rem;flex:1;min-width:14rem;">
       No managed SSH key yet — Steward needs one to log into hosts as <code>steward</code>.
     </span>
-    <form method="post" action="/settings/ansible/generate-key" style="margin:0;">
+    <form method="post" action="/settings/ansible/generate-key" style="margin:0;">{{ csrf_field() }}
       <input type="hidden" name="next" value="/plugins/host_agent/fleet/">
       <button type="submit" class="btn btn-sm">Generate managed key</button>
     </form>
@@ -85,7 +110,7 @@
     agent. Run this once per host.
   </p>
   <form method="post" action="/plugins/host_agent/provision"
-        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
+        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
     <div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
     <div class="form-group" style="margin-bottom:0;">
       <label>Bootstrap user</label>
@@ -111,7 +136,7 @@
     mints a fresh token, and installs the agent. No <code>curl | sh</code> needed.
   </p>
   <form method="post" action="/plugins/host_agent/deploy"
-        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
+        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
     <div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
     <div class="form-group" style="margin-bottom:0;">
       <label>Report interval (s)</label>
@@ -129,7 +154,7 @@
     so identity is preserved.
   </p>
   <form method="post" action="/plugins/host_agent/update"
-        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
+        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
     <div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
     <button type="submit" class="btn">Update</button>
   </form>
@@ -137,39 +162,4 @@
 {% endif %}
 {% endif %}
 
-<div class="card-flush">
-  <table class="table">
-    <thead>
-      <tr>
-        <th>Host</th><th>Reported IP</th><th>Agent version</th><th>Distro</th>
-        <th>Last seen</th><th class="td-actions">Actions</th>
-      </tr>
-    </thead>
-    <tbody>
-      {% for item in registrations %}
-      <tr>
-        <td>{{ item.host.name if item.host else item.reg.host_id }}</td>
-        <td>{{ item.reg.host_ip or "—" }}</td>
-        <td>{{ item.reg.agent_version or "—" }}</td>
-        <td>{{ item.reg.distro or "—" }}</td>
-        <td>{{ item.reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") if item.reg.last_seen_at else "never" }}</td>
-        <td class="td-actions">
-          <form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/rotate-token"
-                style="display:inline;"
-                onsubmit="return confirm('Rotate the token for this host? The existing agent will stop working until reinstalled with the new token.');">
-            <button type="submit" class="btn btn-ghost btn-sm">Rotate token</button>
-          </form>
-          <form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/delete"
-                style="display:inline;"
-                onsubmit="return confirm('Delete this host registration? The agent will be unable to report metrics until re-registered.');">
-            <button type="submit" class="btn btn-danger btn-sm">Delete</button>
-          </form>
-        </td>
-      </tr>
-      {% else %}
-      <tr><td colspan="6" class="empty">No hosts registered. Add one above.</td></tr>
-      {% endfor %}
-    </tbody>
-  </table>
-</div>
 {% endblock %}

plugins/host_agent/templates/widget_history.html

@@ -16,16 +16,14 @@
      toggle that re-requests this widget without entering edit mode. #}
   <div style="flex:0 0 auto;display:flex;align-items:center;gap:0.5rem;flex-wrap:wrap;font-size:0.75rem;color:var(--text-muted);margin-bottom:0.3rem;">
     {% if session.user_id %}
-    <a href="/hosts/{{ host.id }}" style="font-weight:600;">{{ host.name }}</a>
+    <a href="/hosts/{{ host.id }}" style="font-weight:500;">{{ host.name }}</a>
     {% else %}
     <strong style="color:var(--text);">{{ host.name }}</strong>
     {% endif %}
-    <span class="hist-range" style="margin-left:auto;display:inline-flex;border:1px solid var(--border-mid);border-radius:5px;overflow:hidden;">
+    <span class="hist-range" style="margin-left:auto;display:inline-flex;gap:0.25rem;">
       {% for opt in [1, 6, 24] %}
-      <button type="button" onclick="histRange({{ wid }}, '{{ host.id }}', {{ opt }})"
-        style="border:0;cursor:pointer;padding:0.1rem 0.45rem;font-size:0.72rem;
-               background:{% if opt == hours %}var(--accent){% else %}transparent{% endif %};
-               color:{% if opt == hours %}#fff{% else %}var(--text-muted){% endif %};">{{ opt }}h</button>
+      <button type="button" onclick="histRange({{ wid|int }}, {{ opt|int }})"
+        class="range-btn{% if opt == hours %} active{% endif %}">{{ opt }}h</button>
       {% endfor %}
     </span>
   </div>
@@ -36,11 +34,15 @@
 <script>
 // Live time-range switch. Rewrites the parent cell's hx-get so the choice sticks
 // across polls (htmx re-reads the attribute each poll), then fetches immediately.
-window.histRange = function (wid, hostId, hours) {
+// Only `hours` changes: the rest of the query (host, widget id, a share link's
+// token) is kept, so the switch also works on a shared dashboard.
+window.histRange = function (wid, hours) {
   var cell = document.getElementById("widget-" + wid);
   if (!cell) return;
-  var base = (cell.getAttribute("hx-get") || "").split("?")[0];
-  var url = base + "?host_id=" + encodeURIComponent(hostId) + "&hours=" + hours + "&wid=" + wid;
+  var parts = (cell.getAttribute("hx-get") || "").split("?");
+  var query = new URLSearchParams(parts[1] || "");
+  query.set("hours", hours);
+  var url = parts[0] + "?" + query.toString();
   cell.setAttribute("hx-get", url);
   if (window.htmx) window.htmx.ajax("GET", url, { target: cell, swap: "innerHTML" });
 };
@@ -53,9 +55,9 @@ window.histRange = function (wid, hostId, hours) {
     return ("0" + d.getHours()).slice(-2) + ":" + ("0" + d.getMinutes()).slice(-2);
   };
   var ds = [
-    { key: "cpu_pct",      label: "CPU %",    color: "#c8a840" },
-    { key: "mem_used_pct", label: "Mem %",    color: "#4aa86a" },
-    { key: "disk_root",    label: "Disk / %", color: "#c87840" },
+    { key: "cpu_pct",      label: "CPU %",    color: stewardColor("--gold") },
+    { key: "mem_used_pct", label: "Mem %",    color: stewardColor("--green") },
+    { key: "disk_root",    label: "Disk / %", color: stewardColor("--yellow") },
   ];
   new Chart(el.getContext("2d"), {
     type: "line",
@@ -71,14 +73,14 @@ window.histRange = function (wid, hostId, hours) {
       interaction: { mode: "index", intersect: false },
       elements: { point: { radius: 0 } },
       scales: {
-        x: { type: "linear", ticks: { callback: function (v) { return fmtTime(v); }, maxTicksLimit: 6, color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
+        x: { type: "linear", ticks: { callback: function (v) { return fmtTime(v); }, maxTicksLimit: 6, font: { size: 10 } } },
         // Fixed 0–{{ y_max }} ceiling (server snaps the data peak up to the next
         // 10% band). Steady across refreshes — auto-scaling made the zoom jump on
         // every poll — while a banded ceiling still keeps the lines filling the panel.
-        y: { min: 0, max: {{ y_max }}, ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
+        y: { min: 0, max: {{ y_max }}, ticks: { font: { size: 10 } } },
       },
       plugins: {
-        legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } },
+        legend: { labels: { boxWidth: 10, font: { size: 11 } } },
         tooltip: { callbacks: { title: function (items) { return items.length ? fmtTime(items[0].parsed.x) : ""; } } },
       },
     },

plugins/host_agent/templates/widget_table.html

@@ -1,77 +0,0 @@
-{# host_agent widget — fleet glance. One horizontal row per host, zebra-striped
-   so adjacent hosts read as distinct groupings: health dot + host name on the
-   left, metric cells (cpu/mem/disk/load) each with a trend sparkline laid out
-   horizontally on the right. The name wraps (never hard-truncated) but the row
-   stays horizontal. #}
-{% if rows %}
-{% macro metric_cell(label, value, fmt, svg, style="", title="") %}
-<div style="min-width:62px;flex:0 0 auto;" title="{{ title }}">
-  <div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
-  <div style="font-weight:600;font-size:0.8rem;font-variant-numeric:tabular-nums;{{ style }}">
-    {% if value is not none %}{{ fmt|format(value) }}{% else %}—{% endif %}
-  </div>
-  <div style="line-height:0;height:16px;">{{ svg | safe }}</div>
-</div>
-{% endmacro %}
-{# Human-readable throughput, matching the host-detail page. #}
-{% macro fmt_bps(v) %}
-{%- if v is none -%}—
-{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s
-{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s
-{%- else -%}{{ "%.0f"|format(v) }} B/s
-{%- endif -%}
-{% endmacro %}
-{# Two-line down/up (or read/write) throughput cell. #}
-{% macro io_cell(label, down_sym, down_val, up_sym, up_val, title="") %}
-<div style="min-width:80px;flex:0 0 auto;" title="{{ title }}">
-  <div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
-  <div style="font-size:0.72rem;font-variant-numeric:tabular-nums;line-height:1.3;">
-    <div>{{ down_sym }} {{ fmt_bps(down_val) }}</div>
-    <div>{{ up_sym }} {{ fmt_bps(up_val) }}</div>
-  </div>
-</div>
-{% endmacro %}
-<div class="host-fleet">
-  {% for r in rows %}
-  {% set stale = r.stale %}
-  <div style="display:flex;align-items:center;gap:0.6rem 1rem;flex-wrap:wrap;padding:0.5rem 0.6rem;border-radius:4px;{% if loop.index0 % 2 %}background:var(--bg-elevated);{% endif %}">
-    {# Left — health dot + full host name (wraps, never truncated) + last seen #}
-    <div style="display:flex;align-items:center;gap:0.5rem;flex:1 1 150px;min-width:120px;">
-      <span class="dot {% if stale %}dot-dim{% elif (r.cpu_pct or 0) >= 90 or (r.mem_used_pct or 0) >= 90 or (r.disk_worst or 0) >= 90 %}dot-warn{% else %}dot-up{% endif %}"
-            style="flex-shrink:0;"
-            title="{% if stale %}stale / no recent data{% else %}warns at CPU/memory ≥90% or any disk mount ≥90% (the 'disk /' figure shows root only){% endif %}"></span>
-      <span style="min-width:0;">
-        {% if session.user_id %}
-        <a href="/hosts/{{ r.host.id }}" style="font-weight:600;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</a>
-        {% else %}
-        <span style="font-weight:600;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</span>
-        {% endif %}
-        <span style="display:block;font-size:0.7rem;color:var(--text-dim);">
-          {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }}
-        </span>
-      </span>
-    </div>
-    {# Right — metric cells with value + trend, laid out horizontally #}
-    <div style="display:flex;flex-wrap:wrap;gap:0.4rem 1rem;justify-content:flex-end;">
-      {{ metric_cell("cpu", r.cpu_pct, "%.0f%%", r.sparks.cpu, threshold_style(r.cpu_pct, 'cpu'), "Average CPU utilization") }}
-      {{ metric_cell("mem", r.mem_used_pct, "%.0f%%", r.sparks.mem, threshold_style(r.mem_used_pct, 'mem'), "Memory in use") }}
-      {{ metric_cell("disk /", r.disk_root, "%.0f%%", r.sparks.disk, threshold_style(r.disk_root, 'disk'), "Root filesystem (/) usage") }}
-      {{ metric_cell("load", r.load_1m, "%.2f", r.sparks.load, "", "Load average (1m)") }}
-      {# Extra agent metrics — shown only when the host reports them (VMs/containers
-         often have no temp sensor, etc.) so absent data doesn't clutter the row. #}
-      {% if r.net_rx_bps is not none or r.net_tx_bps is not none %}
-      {{ io_cell("net", "↓", r.net_rx_bps, "↑", r.net_tx_bps, "Network throughput (down / up)") }}
-      {% endif %}
-      {% if r.disk_read_bps is not none or r.disk_write_bps is not none %}
-      {{ io_cell("disk i/o", "rd", r.disk_read_bps, "wr", r.disk_write_bps, "Disk I/O (read / write)") }}
-      {% endif %}
-      {% if r.temp_max is not none %}
-      {{ metric_cell("temp", r.temp_max, "%.0f°C", "", threshold_style(r.temp_max, 'temp'), "Hottest sensor") }}
-      {% endif %}
-    </div>
-  </div>
-  {% endfor %}
-</div>
-{% else %}
-<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No hosts with agent data yet.</p>
-{% endif %}

plugins/snmp/__init__.py

@@ -26,3 +26,9 @@ def get_blueprint():
 def get_nav() -> list[dict]:
     # Surfaces in the sidebar's Infrastructure group (SNMP device readings).
     return [{"label": "SNMP", "href": "/plugins/snmp/"}]
+
+
+def get_alert_metrics() -> dict[str, list[str]]:
+    """SNMP metric names are the OID labels each device is configured with, so
+    the list is empty here and the rule form offers the labels recorded so far."""
+    return {"snmp": []}

plugins/snmp/routes.py

@@ -3,46 +3,43 @@ from __future__ import annotations
 from datetime import datetime, timedelta, timezone
 
 from quart import Blueprint, current_app, render_template, request
-from sqlalchemy import func, select
+from sqlalchemy import select
 
 from steward.auth.middleware import require_role
 from steward.models.users import UserRole
+from steward.core.formatting import human_bytes, human_duration
+from steward.core.metric_series import latest_by_resource
 from steward.models.metrics import PluginMetric
 
 snmp_bp = Blueprint("snmp", __name__, template_folder="templates")
 
 
-async def _latest_readings(db, device_names: list[str]) -> dict[str, dict[str, float]]:
-    """Return {device_name: {label: latest_value}} for all configured devices."""
-    if not device_names:
-        return {}
+async def _latest_readings(db, device_names: list[str]) -> dict[str, dict]:
+    """{device_name: {"readings": {label: latest_value}, "at": newest reading's
+    time}} for the devices that have any readings."""
+    latest = await latest_by_resource(db, "snmp", device_names)
+    return {name: {"readings": {m: r.value for m, r in metrics.items()},
+                   "at": max(r.recorded_at for r in metrics.values())}
+            for name, metrics in latest.items() if metrics}
 
-    # Latest value per (resource_name, metric_name) pair
-    subq = (
-        select(
-            PluginMetric.resource_name,
-            PluginMetric.metric_name,
-            func.max(PluginMetric.recorded_at).label("latest_at"),
-        )
-        .where(PluginMetric.source_module == "snmp")
-        .where(PluginMetric.resource_name.in_(device_names))
-        .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
-        .subquery()
-    )
-    result = await db.execute(
-        select(PluginMetric).join(
-            subq,
-            (PluginMetric.resource_name == subq.c.resource_name)
-            & (PluginMetric.metric_name == subq.c.metric_name)
-            & (PluginMetric.recorded_at == subq.c.latest_at),
-        ).where(PluginMetric.source_module == "snmp")
-    )
-    rows = result.scalars().all()
 
-    out: dict[str, dict[str, float]] = {}
-    for row in rows:
-        out.setdefault(row.resource_name, {})[row.metric_name] = row.value
-    return out
+# A device whose newest reading is older than this many poll intervals is not
+# shown as reachable: the poller has stopped getting answers from it.
+STALE_POLLS = 3
+
+
+def _device_state(entry: dict | None, poll_seconds: int, now: datetime) -> dict:
+    """What the page says about a device's readings: {"readings", "at",
+    "reach"}, reach being "up" (answered within STALE_POLLS polls), "stale"
+    (has readings, but none recent) or "pending" (never answered)."""
+    if not entry:
+        return {"readings": {}, "at": None, "reach": "pending"}
+    fresh = (now - entry["at"]).total_seconds() <= STALE_POLLS * max(int(poll_seconds), 1)
+    return {**entry, "reach": "up" if fresh else "stale"}
+
+
+def _poll_seconds() -> int:
+    return int(current_app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60))
 
 
 async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]:
@@ -75,6 +72,7 @@ async def index():
 
     async with current_app.db_sessionmaker() as db:
         latest = await _latest_readings(db, device_names)
+    now = datetime.now(timezone.utc)
 
     # Merge config + latest readings for the template
     devices = []
@@ -84,7 +82,7 @@ async def index():
             "name": name,
             "host": cfg.get("host", ""),
             "oids": cfg.get("oids", []),
-            "readings": latest.get(name, {}),
+            **_device_state(latest.get(name), _poll_seconds(), now),
         })
 
     poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
@@ -150,39 +148,77 @@ async def host_panel(host_id: str):
             return ""
         names = [d.get("name") or d.get("host", "?") for d in matched]
         latest = await _latest_readings(db, names)
+    now = datetime.now(timezone.utc)
 
     devices = [{
         "name": d.get("name") or d.get("host", "?"),
         "host": d.get("host", ""),
         "oids": d.get("oids", []),
-        "readings": latest.get(d.get("name") or d.get("host", "?"), {}),
+        **_device_state(latest.get(d.get("name") or d.get("host", "?")), _poll_seconds(), now),
         "bound": bool(str(d.get("host_id", "")).strip()
                       or str(d.get("steward_host", "")).strip()),
     } for d in matched]
     return await render_template("snmp/host_panel.html", devices=devices)
 
 
+# Labels by suffix: how a reading reads ("uptime_centisecs" → uptime 12d 4h).
+# Counters (octets, packets) only ever grow, so the raw number says little;
+# a device's key readings prefer the others.
+_COUNTER_SUFFIXES = ("_octets", "_packets", "_pkts", "_errors", "_discards")
+
+
+def snmp_reading(label: str, value: float) -> tuple[str, str]:
+    """(name, value) as a reader would say them: units taken from the label's
+    suffix, underscores as spaces."""
+    if label.endswith("_centisecs"):
+        return label[:-len("_centisecs")].replace("_", " "), human_duration(value / 100)
+    if label.endswith("_pct"):
+        return label[:-len("_pct")].replace("_", " "), f"{value:.0f}%"
+    if label.endswith("_dV"):
+        return label[:-len("_dV")].replace("_", " "), f"{value / 10:.1f} V"
+    if label.endswith("_octets"):
+        return label[:-len("_octets")].replace("_", " "), f"{human_bytes(value)} total"
+    return label.replace("_", " "), f"{value:,.0f}" if float(value).is_integer() else f"{value:.4g}"
+
+
+def key_readings(oids: list[dict], readings: dict, limit: int = 2) -> list[tuple[str, str]]:
+    """The first `limit` readings in config order, gauges before counters."""
+    labels = [o.get("label") or o.get("oid") for o in oids if isinstance(o, dict)]
+    present = [lb for lb in labels if lb in readings]
+    ordered = ([lb for lb in present if not lb.endswith(_COUNTER_SUFFIXES)]
+               + [lb for lb in present if lb.endswith(_COUNTER_SUFFIXES)])
+    return [snmp_reading(lb, readings[lb]) for lb in ordered[:limit]]
+
+
+_REACH_ORDER = {"stale": 0, "pending": 1, "up": 2}
+DEVICES_SHOWN = 10
+
+
 @snmp_bp.get("/widget")
 @require_role(UserRole.viewer)
 async def widget():
+    """The dashboard SNMP widget (#5255), in the widget-shape order: a
+    verdict, devices not answering first with since when, then the rest, each
+    with its status in words and one or two readable readings."""
     devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
     device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
 
     async with current_app.db_sessionmaker() as db:
-        latest = await _latest_readings(db, device_names[:10])
+        latest = await _latest_readings(db, device_names)
+    now = datetime.now(timezone.utc)
 
     devices = []
-    for cfg in devices_cfg[:10]:
+    for cfg in devices_cfg:
         name = cfg.get("name") or cfg.get("host", "?")
-        devices.append({
-            "name": name,
-            "oids": cfg.get("oids", []),
-            "readings": latest.get(name, {}),
-        })
-
+        state = _device_state(latest.get(name), _poll_seconds(), now)
+        devices.append({"name": name, **state,
+                        "key": key_readings(cfg.get("oids", []), state["readings"])})
+    devices.sort(key=lambda d: _REACH_ORDER[d["reach"]])
+    counts = {r: sum(1 for d in devices if d["reach"] == r) for r in _REACH_ORDER}
     return await render_template("snmp/widget.html",
-                                 devices=devices,
-                                 total=len(devices_cfg))
+                                 devices=devices[:DEVICES_SHOWN],
+                                 more=max(0, len(devices) - DEVICES_SHOWN),
+                                 counts=counts, total=len(devices))
 
 
 @snmp_bp.get("/device/<device_name>")

plugins/snmp/scheduler.py

@@ -27,7 +27,7 @@ def make_poll_task(app: "Quart") -> ScheduledTask:
 
 async def _do_poll(app: "Quart") -> None:
     from .poller import poll_device
-    from steward.core.alerts import record_metric
+    from steward.core.alerts import MetricBatch
 
     devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", [])
     if not devices:
@@ -35,6 +35,7 @@ async def _do_poll(app: "Quart") -> None:
 
     async with app.db_sessionmaker() as session:
         async with session.begin():
+            batch = MetricBatch()
             for device in devices:
                 if not isinstance(device, dict):
                     continue
@@ -55,8 +56,7 @@ async def _do_poll(app: "Quart") -> None:
                     continue
 
                 for label, value in readings.items():
-                    await record_metric(
-                        session=session,
+                    batch.add(
                         source_module="snmp",
                         resource_name=name,
                         metric_name=label,
@@ -67,3 +67,4 @@ async def _do_poll(app: "Quart") -> None:
                     logger.debug("SNMP polled %s (%s): %d OID(s)", name, host, len(readings))
                 else:
                     logger.debug("SNMP polled %s (%s): no readings", name, host)
+            await batch.record(session)

plugins/snmp/templates/snmp/device.html

@@ -9,9 +9,7 @@
   <span style="font-size:0.8rem;color:var(--text-dim);">{{ device.host }}</span>
   <div style="margin-left:auto;display:flex;gap:0.5rem;">
     {% for h in [1, 6, 24, 168] %}
-    <a href="?hours={{ h }}"
-       class="btn btn-ghost"
-       style="font-size:0.78rem;padding:0.2rem 0.6rem;{% if hours == h %}background:var(--accent);color:#fff;border-color:var(--accent);{% endif %}">
+    <a href="?hours={{ h }}" class="range-btn{% if hours == h %} active{% endif %}"{% if hours == h %} aria-current="true"{% endif %}>
       {% if h < 24 %}{{ h }}h{% elif h == 24 %}24h{% else %}7d{% endif %}
     </a>
     {% endfor %}
@@ -27,7 +25,8 @@
 (function(){
   const raw = {{ history_json | safe }};
   const labels = {{ oid_labels | tojson }};
-  const palette = ["#6060c0","#e07040","#40b080","#c040c0","#4080e0","#e0c040","#c08040"];
+  // Series colours from the tokens, in an order that keeps neighbours apart.
+  const palette = ["--blue", "--yellow", "--green", "--red", "--gold", "--orange", "--text-muted"].map(stewardColor);
 
   // Collect all unique timestamps, sorted
   const tsSet = new Set();
@@ -66,12 +65,10 @@
       responsive: true,
       interaction: { mode: "index", intersect: false },
       plugins: {
-        legend: { position: "top", labels: { color: "#aaa", boxWidth: 12 } },
-        tooltip: { backgroundColor: "#1e1e2e", titleColor: "#ccc", bodyColor: "#aaa" },
+        legend: { position: "top", labels: { boxWidth: 12 } },
       },
       scales: {
-        x: { ticks: { color: "#888", maxTicksLimit: 12 }, grid: { color: "#2a2a3a" } },
-        y: { ticks: { color: "#888" }, grid: { color: "#2a2a3a" } },
+        x: { ticks: { maxTicksLimit: 12 } },
       },
     },
   });

plugins/snmp/templates/snmp/host_panel.html

@@ -9,14 +9,16 @@
   {% for device in devices %}
   <div style="{% if not loop.last %}margin-bottom:1rem;{% endif %}">
     <div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.5rem;flex-wrap:wrap;">
-      <span style="font-weight:600;font-size:0.92rem;">{{ device.name }}</span>
+      <span style="font-weight:500;font-size:0.92rem;">{{ device.name }}</span>
       <span style="font-size:0.78rem;color:var(--text-dim);font-family:ui-monospace,monospace;">{{ device.host }}</span>
       {% if device.bound %}
       <span title="Explicitly bound to this host (host_id / steward_host)"
             style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:3px;padding:0.05rem 0.4rem;">bound</span>
       {% endif %}
-      {% if device.readings %}
-      <span style="font-size:0.74rem;color:var(--green);">&#9679; reachable</span>
+      {% if device.reach == 'up' %}
+      <span style="font-size:0.74rem;" class="tone-ok">&#9679; reachable · read {{ device.at|ago }}</span>
+      {% elif device.reach == 'stale' %}
+      <span style="font-size:0.74rem;" class="tone-warn" title="The values below are from then">&#9679; no answer since {{ device.at|ago }}</span>
       {% else %}
       <span style="font-size:0.74rem;color:var(--text-dim);">&#9675; no data yet</span>
       {% endif %}
@@ -32,7 +34,7 @@
       <div style="background:var(--bg-elevated);border-radius:5px;padding:0.5rem 0.7rem;border:1px solid var(--border);">
         <div style="font-size:0.68rem;color:var(--text-dim);margin-bottom:0.2rem;font-family:ui-monospace,monospace;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ label }}</div>
         {% if val is not none %}
-        <div style="font-size:1rem;font-weight:600;font-variant-numeric:tabular-nums;">
+        <div style="font-size:1rem;font-weight:500;font-variant-numeric:tabular-nums;">
           {% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}<span style="font-size:0.72rem;color:var(--text-muted);">M</span>
           {% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}<span style="font-size:0.72rem;color:var(--text-muted);">K</span>
           {% else %}{{ "%.4g"|format(val) }}{% endif %}

plugins/snmp/templates/snmp/index.html

@@ -1,56 +1,44 @@
 {# plugins/snmp/templates/snmp/index.html #}
 {% extends "base.html" %}
+{% from "_macros.html" import page_header, status_badge, stat_tile, empty_state %}
 {% block title %}SNMP — Steward{% endblock %}
 {% block content %}
-<div style="display:flex;align-items:center;gap:1rem;margin-bottom:1.5rem;">
-  <h1 class="page-title" style="margin:0;">SNMP Devices</h1>
-  <span style="font-size:0.8rem;color:var(--text-dim);">{{ devices|length }} device(s) configured</span>
-</div>
+{{ page_header("SNMP devices", devices|length ~ " device" ~ ("s" if devices|length != 1 else "") ~ " configured") }}
 
 {% if not devices %}
-<div class="card" style="color:var(--text-muted);text-align:center;padding:2rem;">
-  No SNMP devices configured. Add devices to <code>plugin.yaml</code>.
-</div>
+<div class="card">{{ empty_state("No SNMP devices configured. Add them to the snmp plugin's plugin.yaml.") }}</div>
 {% else %}
 
 {% for device in devices %}
-<div class="card" style="margin-bottom:1rem;">
-  <div style="display:flex;align-items:center;gap:1rem;margin-bottom:1rem;">
-    <h2 style="margin:0;font-size:1rem;font-weight:600;">{{ device.name }}</h2>
-    <span style="font-size:0.78rem;color:var(--text-dim);">{{ device.host }}</span>
-    {% if device.readings %}
-    <span style="font-size:0.75rem;color:var(--green);margin-left:auto;">&#9679; reachable</span>
-    {% else %}
-    <span style="font-size:0.75rem;color:var(--text-dim);margin-left:auto;">&#9675; no data yet</span>
-    {% endif %}
-    <a href="/plugins/snmp/device/{{ device.name }}" class="btn btn-ghost" style="font-size:0.78rem;padding:0.2rem 0.6rem;">History</a>
+<section class="card" aria-labelledby="snmp-{{ loop.index }}">
+  <div class="host-health-head">
+    {#- reach: answered within the last few polls (up), only long ago (stale),
+        or never (pending); routes._device_state. #}
+    {% if device.reach == 'up' %}{{ status_badge('up', 'Reachable') }}
+    {% elif device.reach == 'stale' %}{{ status_badge('stale', 'No answer', title="The last reading is from " ~ device.at.strftime('%Y-%m-%d %H:%M UTC') ~ "; the values below are from then") }}
+    {% else %}{{ status_badge('pending', 'No data yet') }}{% endif %}
+    <h2 id="snmp-{{ loop.index }}" class="section-title" style="margin:0;">{{ device.name }}</h2>
+    <span class="field-hint mono">{{ device.host }}</span>
+    {% if device.at %}<span class="field-hint">read {{ device.at|ago }}</span>{% endif %}
+    <a href="/plugins/snmp/device/{{ device.name }}" class="btn btn-sm btn-ghost" style="margin-left:auto;">History</a>
   </div>
 
   {% if device.readings %}
-  <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(200px,1fr));gap:0.75rem;">
+  <div class="stat-strip" style="margin-bottom:0;">
     {% for oid_cfg in device.oids %}
     {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
     {% set val = device.readings.get(label) %}
-    <div style="background:var(--bg-alt);border-radius:6px;padding:0.6rem 0.9rem;">
-      <div style="font-size:0.72rem;color:var(--text-dim);margin-bottom:0.2rem;font-family:monospace;">{{ label }}</div>
-      {% if val is not none %}
-      <div style="font-size:1.1rem;font-weight:600;font-variant-numeric:tabular-nums;">
-        {% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}<span style="font-size:0.75rem;color:var(--text-muted);">M</span>
-        {% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}<span style="font-size:0.75rem;color:var(--text-muted);">K</span>
-        {% else %}{{ "%.4g"|format(val) }}{% endif %}
-      </div>
-      {% else %}
-      <div style="font-size:0.85rem;color:var(--text-dim);">—</div>
-      {% endif %}
-    </div>
+    {{ stat_tile(label,
+                 "—" if val is none else
+                 ("%.2fM"|format(val / 1000000)) if val >= 1000000 else
+                 ("%.1fK"|format(val / 1000)) if val >= 1000 else
+                 "%.4g"|format(val)) }}
     {% endfor %}
   </div>
   {% else %}
-  <p style="font-size:0.85rem;color:var(--text-muted);margin:0;">
-    No readings yet — waiting for next poll cycle.
-  </p>
+  <p class="field-hint">No readings yet; waiting for the next poll.</p>
   {% endif %}
-</div>
+</section>
 {% endfor %}
 
 {% endif %}

plugins/snmp/templates/snmp/widget.html

@@ -1,49 +1,37 @@
-{# plugins/snmp/templates/snmp/widget.html #}
-{% if devices %}
-{% for device in devices %}
-<div class="ping-row" style="align-items:flex-start;flex-direction:column;gap:0.3rem;padding:0.4rem 0;">
-  <div style="display:flex;align-items:center;gap:0.6rem;width:100%;">
-    <span style="font-size:0.82rem;font-weight:500;flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">
-      {{ device.name }}
-    </span>
-    {% if device.readings %}
-    <span style="font-size:0.7rem;color:var(--green);">&#9679;</span>
-    {% else %}
-    <span style="font-size:0.7rem;color:var(--text-dim);">&#9675;</span>
-    {% endif %}
-    <a href="/plugins/snmp/device/{{ device.name }}" style="font-size:0.72rem;color:var(--text-muted);">detail →</a>
-  </div>
-  {% if device.readings %}
-  <div style="display:flex;flex-wrap:wrap;gap:0.5rem 1rem;padding-left:0.1rem;">
-    {% for oid_cfg in device.oids[:4] %}
-    {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
-    {% set val = device.readings.get(label) %}
-    {% if val is not none %}
-    <span style="font-size:0.78rem;font-variant-numeric:tabular-nums;">
-      <span style="color:var(--text-muted);">{{ label }}</span>
-      <span style="margin-left:0.25rem;">
-        {% if val >= 1000000 %}{{ "%.1f"|format(val / 1000000) }}M
-        {% elif val >= 1000 %}{{ "%.0f"|format(val / 1000) }}K
-        {% else %}{{ "%.4g"|format(val) }}{% endif %}
-      </span>
-    </span>
-    {% endif %}
-    {% endfor %}
-    {% if device.oids|length > 4 %}
-    <span style="font-size:0.72rem;color:var(--text-dim);">+{{ device.oids|length - 4 }} more</span>
-    {% endif %}
-  </div>
-  {% endif %}
-</div>
-{% if not loop.last %}
-<div style="height:1px;background:var(--border-low);margin:0.1rem 0;"></div>
-{% endif %}
-{% endfor %}
-{% if total > devices|length %}
-<div style="color:var(--text-dim);font-size:0.75rem;padding:0.25rem 0;">
-  +{{ total - devices|length }} more — <a href="/plugins/snmp/">view all →</a>
-</div>
-{% endif %}
+{# plugins/snmp/templates/snmp/widget.html — the dashboard SNMP widget (#5255):
+   a verdict, devices that stopped answering first with since when, then the
+   rest, each with its status in words and its key readings, named and with
+   units. The device name links to its page. #}
+{% from "_macros.html" import status_dot %}
+{% if not total %}
+<div class="w-verdict" style="color:var(--text-muted);">No SNMP devices configured.</div>
 {% else %}
-<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No SNMP devices configured.</p>
+{% set stale = devices | selectattr("reach", "equalto", "stale") | list %}
+<div class="w-verdict {{ 'tone-crit' if counts.stale else ('tone-warn' if counts.pending else 'tone-ok') }}">
+  {%- if counts.stale == 1 and stale %}{{ stale[0].name }}: no reply since {{ stale[0].at|ago }}.
+  {%- elif counts.stale %}{{ counts.stale }} of {{ total }} devices not answering.
+  {%- elif counts.pending %}{{ counts.pending }} not polled yet, {{ counts.up }} answering.
+  {%- else %}All {{ total }} device{{ 's' if total != 1 }} answering.{% endif -%}
+</div>
+<table class="w-table">
+  <colgroup><col style="width:40%"><col><col class="w-wide-only" style="width:6rem"></colgroup>
+  <thead><tr><th>Device</th><th>Readings</th><th class="w-wide-only">Polled</th></tr></thead>
+  <tbody>
+  {% for d in devices %}
+  <tr>
+    <td class="w-name" title="{{ d.name }}">
+      {{- status_dot('down' if d.reach == 'stale' else ('pending' if d.reach == 'pending' else 'up')) -}}
+      {% if session.user_id %}<a href="/plugins/snmp/device/{{ d.name }}">{{ d.name }}</a>{% else %}{{ d.name }}{% endif %}
+      {%- if d.reach == 'stale' %}<span class="w-sub tone-crit">no reply since {{ d.at|ago }}; readings are from then</span>
+      {%- elif d.reach == 'pending' %}<span class="w-sub">no data yet</span>{% endif %}
+    </td>
+    <td title="{% for n, v in d.key %}{{ n }} {{ v }}{% if not loop.last %} · {% endif %}{% endfor %}">
+      {%- for n, v in d.key %}<span style="color:var(--text-muted);">{{ n }}</span> {{ v }}{% if not loop.last %} · {% endif %}{% else %}—{% endfor -%}
+    </td>
+    <td class="w-wide-only">{% if d.at %}{{ d.at|ago }}{% else %}—{% endif %}</td>
+  </tr>
+  {% endfor %}
+  </tbody>
+</table>
+{% if more %}<div class="w-more">+{{ more }} more{% if session.user_id %} · <a href="/plugins/snmp/">all devices</a>{% endif %}</div>{% endif %}
 {% endif %}

plugins/traefik/__init__.py

@@ -12,6 +12,11 @@ def setup(app: "Quart") -> None:
     global _app
     _app = app
     from .models import TraefikMetric, TraefikCert, TraefikGlobalStat, TraefikAccessSummary  # noqa: F401
+    from steward.core.cleanup import prune_by_age
+    # Samples older than data.retention_days are pruned by the cleanup (#5155).
+    prune_by_age(TraefikMetric.scraped_at)
+    prune_by_age(TraefikGlobalStat.scraped_at)
+    prune_by_age(TraefikAccessSummary.period_end)
 
 
 def get_scheduled_tasks() -> list:
@@ -33,3 +38,11 @@ def get_blueprint():
 def get_nav() -> list[dict]:
     # Surfaces in the sidebar's Infrastructure group (Traefik services view).
     return [{"label": "Traefik", "href": "/plugins/traefik/"}]
+
+
+def get_alert_metrics() -> dict[str, list[str]]:
+    """What scheduler.py records for alert rules: per router (resource = the
+    router name) and cert_expiry_days per certificate (resource = its CN)."""
+    return {"traefik": ["request_rate", "error_rate_4xx_pct", "error_rate_5xx_pct",
+                        "latency_p50_ms", "latency_p95_ms", "latency_p99_ms",
+                        "response_bytes_rate", "cert_expiry_days"]}

plugins/traefik/models.py

@@ -1,16 +1,16 @@
 # plugins/traefik/models.py
 from __future__ import annotations
-import uuid
 from datetime import datetime, timezone
 from sqlalchemy import DateTime, Float, Integer, String, Text
 from sqlalchemy.orm import Mapped, mapped_column
 from steward.models.base import Base
+from steward.core.ids import time_ordered_id
 
 
 class TraefikMetric(Base):
     __tablename__ = "traefik_metrics"
 
-    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
     router_name: Mapped[str] = mapped_column(String(255), nullable=False)
     scraped_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
@@ -41,7 +41,7 @@ class TraefikAccessSummary(Base):
     """Aggregated traffic-origin summary — one row per scheduler tick."""
     __tablename__ = "traefik_access_summaries"
 
-    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
     period_start: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
     period_end: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
     total_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
@@ -57,7 +57,7 @@ class TraefikGlobalStat(Base):
     """One row per scrape — process-level and config stats."""
     __tablename__ = "traefik_global_stats"
 
-    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
     scraped_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
     )

plugins/traefik/plugin.yaml

@@ -14,6 +14,9 @@ tags:
 
 config:
   metrics_url: "http://localhost:8080/metrics"
+  # Traefik's API, for which hosts each router serves (so a certificate's
+  # expiry shows on its router). Empty: /api on the metrics URL's host.
+  api_url: ""
   scrape_interval_seconds: 60
   access_log:
     enabled: false

plugins/traefik/routes.py

@@ -1,15 +1,18 @@
 # plugins/traefik/routes.py
 from __future__ import annotations
 from collections import defaultdict
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from quart import Blueprint, current_app, render_template, request
-from sqlalchemy import Integer, cast, func, select
+from sqlalchemy import func, select, text
 import json
 
 from steward.auth.middleware import require_role
 from steward.models.users import UserRole
 from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample
+from steward.core.formatting import cert_left, status_class
 from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric
+from .scheduler import router_domains
+from .scraper import cert_names, covers
 
 traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
 
@@ -36,14 +39,130 @@ def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
     )
 
 
+# A router serving any real share of 5xx is failing; a 4xx share above this
+# is worth a look (clients hitting missing routes, auth failures).
+ERROR_5XX_PCT = 0.1
+ERROR_4XX_PCT = 1.0
+_GRADE_ORDER = {"crit": 0, "warn": 1, "ok": 2}
+
+
+def router_state(latest, thresholds: dict) -> tuple[str, str | None]:
+    """(grade, why) for a router's latest scrape: crit while it serves 5xx,
+    warn on a p95 past the latency critical cutoff (Settings → Thresholds)
+    or a high 4xx share, otherwise ok with nothing to say."""
+    if (latest.error_rate_5xx_pct or 0) > ERROR_5XX_PCT:
+        return "crit", f"{latest.error_rate_5xx_pct:.1f}% of responses are 5xx"
+    p95 = latest.latency_p95_ms
+    if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
+        return "warn", f"p95 latency is {p95:.0f} ms"
+    if (latest.error_rate_4xx_pct or 0) > ERROR_4XX_PCT:
+        return "warn", f"{latest.error_rate_4xx_pct:.1f}% of responses are 4xx"
+    return "ok", None
+
+
+# A router missing from this many scrapes in a row is no longer reported:
+# Traefik dropped it (a removed service), so its last state is history.
+GONE_AFTER_SCRAPES = 3
+
+
+def gone_routers(routers: list[dict], scrape_seconds: int) -> tuple[list[dict], list[dict]]:
+    """(current, gone): a router whose newest row trails the newest scrape of
+    any router by GONE_AFTER_SCRAPES intervals is gone, newest-seen first."""
+    if not routers:
+        return [], []
+    newest = max(r["latest"].scraped_at for r in routers)
+    cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
+    current = [r for r in routers if r["latest"].scraped_at >= cutoff]
+    gone = sorted((r for r in routers if r["latest"].scraped_at < cutoff),
+                  key=lambda r: r["latest"].scraped_at, reverse=True)
+    return current, gone
+
+
+def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
+    """(problems, healthy): failing routers before slow or noisy ones, each
+    group by name; healthy routers by name."""
+    problems = sorted((r for r in routers if r["state"] != "ok"),
+                      key=lambda r: (_GRADE_ORDER[r["state"]], r["name"].lower()))
+    healthy = sorted((r for r in routers if r["state"] == "ok"), key=lambda r: r["name"].lower())
+    return problems, healthy
+
+
+# Newest row per router. Postgres has no skip scan, so DISTINCT or max() GROUP
+# BY reads every row in retention; this walks the (router_name, scraped_at)
+# index one router at a time instead: a step per router, not per row.
+_LATEST_PER_ROUTER = text("""
+    WITH RECURSIVE names AS (
+        SELECT min(router_name) AS name FROM traefik_metrics
+        UNION ALL
+        SELECT (SELECT min(router_name) FROM traefik_metrics WHERE router_name > names.name)
+        FROM names WHERE names.name IS NOT NULL
+    )
+    SELECT m.* FROM names
+    JOIN LATERAL (
+        SELECT * FROM traefik_metrics WHERE router_name = names.name
+        ORDER BY scraped_at DESC LIMIT 1
+    ) m ON true
+""")
+
+
+async def latest_per_router(db) -> list:
+    return list((await db.execute(
+        select(TraefikMetric).from_statement(_LATEST_PER_ROUTER))).scalars().all())
+
+
+def current_certs(certs, scrape_seconds: int) -> tuple[list, list]:
+    """(served, old): a certificate is served while Traefik still reports it.
+    Rows are upserted by serial and never deleted, so a renewed or retired
+    certificate lingers; one missing from the last GONE_AFTER_SCRAPES
+    scrapes is old (#5254)."""
+    certs = list(certs)
+    if not certs:
+        return [], []
+    newest = max(c.scraped_at for c in certs)
+    cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
+    return ([c for c in certs if c.scraped_at >= cutoff],
+            sorted((c for c in certs if c.scraped_at < cutoff), key=lambda c: c.scraped_at, reverse=True))
+
+
+def router_hosts(name: str, domains: dict[str, list[str]]) -> list[str]:
+    """The hosts a metrics router name serves. The API names a router
+    "app@docker"; metrics may carry it as "websecure-app@docker", one per
+    entrypoint, so a name not found as-is is tried without that prefix."""
+    if name in domains:
+        return domains[name]
+    for api_name, hosts in domains.items():
+        if name.endswith("-" + api_name):
+            return hosts
+    return []
+
+
+def certs_by_router(routers: list[str], domains: dict[str, list[str]], certs, now) -> dict[str, dict]:
+    """{router: {"cn", "days"}}: the soonest-expiring served certificate valid
+    for one of the router's hosts. A router with no hosts, or none covered,
+    is absent."""
+    out: dict[str, dict] = {}
+    names = [(c, cert_names(c.cn, c.sans)) for c in certs]
+    for router in routers:
+        for host in router_hosts(router, domains):
+            for c, cn in names:
+                if covers(cn, host):
+                    days = (c.not_after - now).total_seconds() / 86400.0
+                    if router not in out or days < out[router]["days"]:
+                        out[router] = {"cn": c.cn or host, "days": days}
+    return out
+
+
+def _access_log() -> dict:
+    """The plugin's access_log settings ({} when unset or malformed)."""
+    cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
+    return cfg if isinstance(cfg, dict) else {}
+
+
 @traefik_bp.get("/")
 @require_role(UserRole.viewer)
 async def index():
     poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
-    _al_cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
-    if not isinstance(_al_cfg, dict):
-        _al_cfg = {}
-    access_log_enabled = _al_cfg.get("enabled", False)
+    access_log_enabled = bool(_access_log().get("enabled", False))
     current_range = request.args.get("range", DEFAULT_RANGE)
     return await render_template(
         "traefik/index.html",
@@ -59,20 +178,15 @@ async def rows():
     """HTMX fragment: service cards with sparklines scoped to selected time range."""
     since, range_key = parse_range(request.args.get("range"))
     now = datetime.now(timezone.utc)
+    thresholds = current_app.config.get("THRESHOLDS", {})
 
     async with current_app.db_sessionmaker() as db:
-        # All known routers
-        result = await db.execute(
-            select(TraefikMetric.router_name)
-            .distinct()
-            .order_by(TraefikMetric.router_name)
-        )
-        routers = [row[0] for row in result.all()]
+        # Every known router, with its most recent raw row
+        newest = {m.router_name: m for m in await latest_per_router(db)}
+        routers = sorted(newest)
 
         b_secs = bucket_seconds(since)
-        bucket_col = (
-            cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs
-        ).label("bucket")
+        bucket_col = _time_bucket(TraefikMetric.scraped_at, b_secs)
 
         router_data = []
         for router in routers:
@@ -92,20 +206,14 @@ async def rows():
             )
             history = result.all()
 
-            # Latest value — always the most recent raw row
-            result = await db.execute(
-                select(TraefikMetric)
-                .where(TraefikMetric.router_name == router)
-                .order_by(TraefikMetric.scraped_at.desc())
-                .limit(1)
-            )
-            latest = result.scalar_one_or_none()
-            if not latest:
-                continue
+            latest = newest[router]
 
+            state, why = router_state(latest, thresholds)
             router_data.append({
                 "name": router,
                 "latest": latest,
+                "state": state,
+                "why": why,
                 "sparkline_req": _sparkline([r.request_rate for r in history]),
                 "sparkline_p95": _sparkline([r.latency_p95_ms for r in history]),
                 "sparkline_5xx": _sparkline([r.error_rate_5xx_pct for r in history]),
@@ -125,22 +233,39 @@ async def rows():
         )
         raw_certs = result.scalars().all()
 
-    certs = []
-    for c in raw_certs:
-        days = (c.not_after - now).total_seconds() / 86400.0
-        certs.append({
+    scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
+    current, gone = gone_routers(router_data, scrape)
+    served, old = current_certs(raw_certs, scrape)
+    domains = router_domains()
+
+    def cert_row(c):
+        names = cert_names(c.cn, c.sans)
+        return {
             "serial": c.serial,
             "cn": c.cn or c.serial,
-            "sans": c.sans,
+            "sans": [n for n in names if n != c.cn],
             "not_after": c.not_after,
-            "days_remaining": days,
-        })
+            "days_remaining": (c.not_after - now).total_seconds() / 86400.0,
+            # The routers whose hosts it covers; None when Traefik's API
+            # couldn't say which hosts each router serves.
+            "routers": None if domains is None else sorted(
+                {r["name"].split("@")[0] for r in current
+                 for h in router_hosts(r["name"], domains) if covers(names, h)}),
+        }
 
+    certs = [cert_row(c) for c in served]
+    old_certs = [cert_row(c) for c in old]
+    problems, healthy = split_routers(current)
     return await render_template(
         "traefik/rows.html",
-        router_data=router_data,
+        problems=problems,
+        healthy=healthy,
+        gone=gone,
+        error_4xx=ERROR_4XX_PCT,
+        error_5xx=ERROR_5XX_PCT,
         global_stat=global_stat,
         certs=certs,
+        old_certs=old_certs,
         range_key=range_key,
     )
 
@@ -148,53 +273,53 @@ async def rows():
 @traefik_bp.get("/widget")
 @require_role(UserRole.viewer)
 async def widget():
-    """HTMX dashboard widget fragment."""
+    """The dashboard Routers widget (#5254), in the widget-shape order: a
+    verdict, routers in trouble with why, then the busiest healthy ones under
+    one header. Only routers and certificates Traefik still reports. A
+    certificate's expiry shows on the router it serves, and makes that router
+    a problem once it is inside the Settings → Thresholds TLS window."""
     now = datetime.now(timezone.utc)
+    thresholds = current_app.config.get("THRESHOLDS", {})
+    scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
 
     async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(TraefikMetric.router_name).distinct().order_by(TraefikMetric.router_name)
-        )
-        routers = [row[0] for row in result.all()]
-        router_data = []
-        for router in routers:
-            result = await db.execute(
-                select(TraefikMetric)
-                .where(TraefikMetric.router_name == router)
-                .order_by(TraefikMetric.scraped_at.desc())
-                .limit(1)
-            )
-            latest = result.scalar_one_or_none()
-            if latest:
-                router_data.append({"name": router, "latest": latest})
-
-        router_data.sort(key=lambda r: (
-            -(1 if r["latest"].error_rate_5xx_pct > 0.1 else 0),
-            -(1 if r["latest"].error_rate_4xx_pct > 5.0 else 0),
-            -r["latest"].request_rate,
-        ))
-        total_routers = len(router_data)
-        router_data = router_data[:5]
-
-        result = await db.execute(
-            select(TraefikCert).order_by(TraefikCert.not_after)
-        )
-        raw_certs = result.scalars().all()
-
-    expiring_certs = []
-    for c in raw_certs:
-        days = (c.not_after - now).total_seconds() / 86400.0
-        if days < 30:
-            expiring_certs.append({"cn": c.cn or c.serial, "days_remaining": days})
+        latest = await latest_per_router(db)
+        certs = (await db.execute(select(TraefikCert))).scalars().all()
 
+    current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
+    served, _old = current_certs(certs, scrape)
+    domains = router_domains()
+    by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
+    for r in current:
+        r["state"], r["why"] = router_state(r["latest"], thresholds)
+        r["cert"] = by_router.get(r["name"])
+        if r["cert"]:
+            grade = status_class(r["cert"]["days"], "tls_days", thresholds)
+            r["cert"]["grade"] = grade
+            if _GRADE_ORDER.get(grade, 2) < _GRADE_ORDER[r["state"]]:
+                left = cert_left(r["cert"]["days"])     # "12 days", "expires today", "expired …"
+                r["state"], r["why"] = grade, (
+                    f"certificate expires in {left}" if left[0].isdigit() else f"certificate {left}")
+    problems, healthy = split_routers(current)
+    healthy.sort(key=lambda r: -(r["latest"].request_rate or 0))
+    # With no router domains to go on, certificates can't sit on their
+    # routers: the expiring ones are listed after the routers instead.
+    loose = [] if domains is not None else sorted(
+        ({"cn": c.cn or c.serial, "days": (c.not_after - now).total_seconds() / 86400.0} for c in served),
+        key=lambda c: c["days"])
+    loose = [c for c in loose if status_class(c["days"], "tls_days", thresholds) in ("warn", "crit")]
     return await render_template(
         "traefik/widget.html",
-        router_data=router_data,
-        expiring_certs=expiring_certs,
-        total_routers=total_routers,
+        problems=problems, healthy=healthy[:ROUTERS_SHOWN],
+        healthy_more=max(0, len(healthy) - ROUTERS_SHOWN),
+        total=len(current), loose_certs=loose, domains_known=domains is not None,
     )
 
 
+# Healthy routers on the dashboard widget (the busiest); the page lists all.
+ROUTERS_SHOWN = 6
+
+
 @traefik_bp.get("/widget/access_log")
 @require_role(UserRole.viewer)
 async def widget_access_log():
@@ -248,38 +373,66 @@ async def widget_access_log():
                                  summary=summary, widget_id=widget_id)
 
 
+def _time_bucket(column, seconds: int):
+    """The start of the `seconds`-wide window a timestamp falls in, as
+    host_agent/metrics_query.py buckets. Integer epoch / int rendered as
+    numeric division (no two scrapes shared a bucket) and overflowed int4
+    after 2038 (#5256)."""
+    return func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, int(seconds)), column,
+                         func.to_timestamp(0)).label("bucket")
+
+
+async def request_history(db, since, bucket_seconds: int) -> list[dict]:
+    """[{"at", "req_rate", "err_pct"}] per time bucket since `since`: each
+    scrape summed over routers (the total req/s), averaged over the bucket;
+    the 5xx share weighted by each router's traffic, so a quiet router's
+    errors don't count as much as a busy one's."""
+    per_scrape = (
+        select(
+            TraefikMetric.scraped_at.label("at"),
+            func.sum(TraefikMetric.request_rate).label("req"),
+            func.sum(TraefikMetric.request_rate * TraefikMetric.error_rate_5xx_pct).label("err_weighted"),
+        )
+        .where(TraefikMetric.scraped_at >= since)
+        .group_by(TraefikMetric.scraped_at)
+        .subquery()
+    )
+    bucket_col = _time_bucket(per_scrape.c.at, bucket_seconds)
+    rows = (await db.execute(
+        select(
+            func.avg(per_scrape.c.req).label("req_rate"),
+            func.sum(per_scrape.c.err_weighted).label("err_weighted"),
+            func.sum(per_scrape.c.req).label("req_sum"),
+            func.min(per_scrape.c.at).label("at"),
+            bucket_col,
+        )
+        .group_by(bucket_col)
+        .order_by(bucket_col)
+    )).all()
+    return [{"at": r.at, "req_rate": round(r.req_rate or 0, 3),
+             "err_pct": round((r.err_weighted or 0) / r.req_sum, 2) if r.req_sum else 0.0}
+            for r in rows]
+
+
 @traefik_bp.get("/widget/request_chart")
 @require_role(UserRole.viewer)
 async def widget_request_chart():
-    """HTMX dashboard widget: Chart.js request rate + error % over time."""
-    from datetime import timedelta
+    """HTMX dashboard widget: requests per second across all routers, and the
+    share of them answered 5xx, over the last few hours (#5256). Each scrape
+    is summed over routers first (the total), then averaged per bucket; the
+    5xx share is weighted by each router's traffic, not averaged per router."""
     hours = max(1, min(24, int(request.args.get("hours", 6) or 6)))
     widget_id = request.args.get("wid", "0")
 
     since = datetime.now(timezone.utc) - timedelta(hours=hours)
     b_secs = max(60, (hours * 3600) // 80)  # ~80 buckets
 
-    bucket_col = (
-        cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs
-    ).label("bucket")
-
     async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(
-                func.avg(TraefikMetric.request_rate).label("req_rate"),
-                func.avg(TraefikMetric.error_rate_5xx_pct).label("err_pct"),
-                func.min(TraefikMetric.scraped_at).label("scraped_at"),
-                bucket_col,
-            )
-            .where(TraefikMetric.scraped_at >= since)
-            .group_by(bucket_col)
-            .order_by(bucket_col)
-        )
-        history = result.all()
+        history = await request_history(db, since, b_secs)
 
-    labels = list(range(len(history)))
-    req_rates = [round(r.req_rate or 0, 3) for r in history]
-    err_pcts = [round(r.err_pct or 0, 2) for r in history]
+    labels = [int(r["at"].timestamp() * 1000) for r in history]   # epoch ms; the browser shows local time
+    req_rates = [r["req_rate"] for r in history]
+    err_pcts = [r["err_pct"] for r in history]
 
     return await render_template(
         "traefik/widget_request_chart.html",
@@ -288,6 +441,9 @@ async def widget_request_chart():
         error_rate_json=json.dumps(err_pcts),
         widget_id=widget_id,
         hours=hours,
+        now_req=req_rates[-1] if req_rates else None,
+        peak_req=max(req_rates) if req_rates else None,
+        peak_err=max(err_pcts) if err_pcts else None,
     )
 
 
@@ -308,8 +464,10 @@ async def traffic():
         rows = result.scalars().all()
 
     if not rows:
+        al = _access_log()
         return await render_template(
-            "traefik/traffic.html", summary=None, history=[], range_key=range_key
+            "traefik/traffic.html", summary=None, history=[], range_key=range_key,
+            access_log_enabled=bool(al.get("enabled", False)), access_log_path=al.get("path", ""),
         )
 
     total = sum(r.total_requests for r in rows)

plugins/traefik/scheduler.py

@@ -16,6 +16,16 @@ logger = logging.getLogger(__name__)
 _prev_metrics: dict = {}
 _prev_time: float = 0.0
 
+# Router name -> host names from Traefik's API, refreshed each scrape (#5254).
+# None until the API has answered once, and again whenever it stops answering,
+# so readers can tell "no domains" from "don't know".
+_router_domains: dict[str, list[str]] | None = None
+
+
+def router_domains() -> dict[str, list[str]] | None:
+    return _router_domains
+
+
 # Access log state
 _geoip_db = None
 _access_log_last_tick: "datetime | None" = None
@@ -38,17 +48,19 @@ def make_scrape_task(app: "Quart") -> ScheduledTask:
 
 async def _do_scrape(app: "Quart") -> None:
     """Fetch Traefik metrics, write to DB, and emit plugin_metrics."""
-    global _prev_metrics, _prev_time
+    global _prev_metrics, _prev_time, _router_domains
 
     from datetime import datetime, timezone
     from .models import TraefikCert, TraefikGlobalStat, TraefikMetric
     from .scraper import (
+        api_base,
         compute_global_stats,
         compute_router_metrics,
         extract_certs,
         fetch_metrics,
+        fetch_router_domains,
     )
-    from steward.core.alerts import record_metric
+    from steward.core.alerts import MetricBatch
 
     metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"]
 
@@ -57,6 +69,13 @@ async def _do_scrape(app: "Quart") -> None:
     except Exception:
         logger.exception("Traefik scrape failed (url=%s)", metrics_url)
         return
+    cfg = app.config["PLUGINS"]["traefik"]
+    try:
+        _router_domains = await fetch_router_domains(api_base(metrics_url, cfg.get("api_url") or ""))
+    except Exception as exc:
+        if _router_domains is not None:
+            logger.warning("Traefik API stopped answering (%s); certificates shown without routers", exc)
+        _router_domains = None
 
     now = time.monotonic()
     elapsed = now - _prev_time if _prev_time else 60.0
@@ -71,6 +90,7 @@ async def _do_scrape(app: "Quart") -> None:
 
     async with app.db_sessionmaker() as session:
         async with session.begin():
+            batch = MetricBatch()
             # ── Per-service metrics ───────────────────────────────────────────
             for router_name, metrics in router_metrics.items():
                 row = TraefikMetric(
@@ -95,8 +115,7 @@ async def _do_scrape(app: "Quart") -> None:
                     ("latency_p99_ms", metrics["latency_p99_ms"]),
                     ("response_bytes_rate", metrics["response_bytes_rate"]),
                 ]:
-                    await record_metric(
-                        session=session,
+                    batch.add(
                         source_module="traefik",
                         resource_name=router_name,
                         metric_name=metric_name,
@@ -125,14 +144,15 @@ async def _do_scrape(app: "Quart") -> None:
 
                 # Emit cert_expiry_days for alert rules
                 days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0
-                await record_metric(
-                    session=session,
+                batch.add(
                     source_module="traefik",
                     resource_name=cert["cn"] or cert["serial"],
                     metric_name="cert_expiry_days",
                     value=max(0.0, days_remaining),
                 )
 
+            await batch.record(session)
+
     logger.debug(
         "Traefik scrape: %d router(s), %d cert(s)",
         len(router_metrics),

plugins/traefik/scraper.py

@@ -274,3 +274,69 @@ async def fetch_metrics(metrics_url: str) -> ParsedMetrics:
         response = await client.get(metrics_url)
         response.raise_for_status()
     return parse_prometheus(response.text)
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Router domains (Traefik API)
+# ──────────────────────────────────────────────────────────────────────────────
+#
+# The metrics name each router but not the hosts it answers for, so which
+# certificate serves which router is only knowable from Traefik's API. Read
+# once per scrape; when the API isn't reachable, routers simply have no
+# domains and certificates are shown on their own (#5254).
+
+_HOST_MATCHER = re.compile(r"Host(?:SNI)?\(([^)]*)\)")
+_QUOTED = re.compile(r"[`\"']([^`\"']+)[`\"']")
+
+
+def rule_hosts(rule: str) -> list[str]:
+    """The host names a router rule matches: Host(`a`, `b`) and HostSNI(`c`)
+    give ["a", "b", "c"]; HostRegexp and path-only rules give nothing."""
+    hosts: list[str] = []
+    for args in _HOST_MATCHER.findall(rule or ""):
+        hosts.extend(h.strip().lower() for h in _QUOTED.findall(args) if h.strip() != "*")
+    return hosts
+
+
+def api_base(metrics_url: str, api_url: str = "") -> str:
+    """The Traefik API's base: the configured api_url, else /api on the
+    metrics URL's own host (the default "traefik" entrypoint serves both)."""
+    if api_url:
+        return api_url.rstrip("/")
+    parts = httpx.URL(metrics_url)
+    return str(parts.copy_with(path="/api", query=None, fragment=None)).rstrip("/")
+
+
+async def fetch_router_domains(base: str) -> dict[str, list[str]]:
+    """{router name: host names} for every enabled HTTP and TCP router."""
+    out: dict[str, list[str]] = {}
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        for kind in ("http", "tcp"):
+            response = await client.get(f"{base}/{kind}/routers", params={"per_page": 1000})
+            if kind == "tcp" and response.status_code == 404:
+                continue
+            response.raise_for_status()
+            for r in response.json():
+                if r.get("status", "enabled") != "enabled" or not r.get("name"):
+                    continue
+                hosts = rule_hosts(r.get("rule", ""))
+                if hosts:
+                    out.setdefault(r["name"], []).extend(hosts)
+    return out
+
+
+def cert_names(cn: str | None, sans: str | None) -> set[str]:
+    """Every name a certificate covers: its CN and SANs, lower-cased."""
+    names = {n.strip().lower() for n in (sans or "").split(",") if n.strip()}
+    if cn:
+        names.add(cn.strip().lower())
+    return names
+
+
+def covers(names: set[str], host: str) -> bool:
+    """Whether a certificate with these names is valid for host, a wildcard
+    "*.example.com" covering one label: "a.example.com", not "a.b.example.com"."""
+    if host in names:
+        return True
+    head, _, rest = host.partition(".")
+    return bool(rest) and f"*.{rest}" in names

plugins/traefik/templates/traefik/rows.html

@@ -1,127 +1,141 @@
-{# plugins/traefik/templates/traefik/rows.html #}
-
-{# ── Global stats bar ──────────────────────────────────────────────────────── #}
-{% if global_stat %}
-<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--card-bg);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;">
-
-  {% if global_stat.open_conns_total is not none %}
-  <span>
-    <span style="color:var(--text-muted);">Open conns</span>
-    <span style="margin-left:0.4rem;font-weight:600;">{{ global_stat.open_conns_total | int }}</span>
-  </span>
-  {% endif %}
+{# traefik/rows.html — HTMX fragment for the Traefik page: routers that need
+   attention first (routes.router_state), the proxy's own counts, the healthy
+   routers folded into one table, then certificates by soonest expiry. #}
+{% from "_macros.html" import stat_tile, status_badge %}
 
+{# ── Counts ───────────────────────────────────────────────────────────────── #}
+<div class="stat-strip">
+  {{ stat_tile("Needs attention", problems|length, state="crit" if problems else "inactive") }}
+  {{ stat_tile("Healthy routers", healthy|length, state="up" if healthy else "inactive") }}
+  {% if global_stat %}
   {% if global_stat.config_last_reload_success is not none %}
-  <span>
-    <span style="color:var(--text-muted);">Config</span>
-    {% if global_stat.config_last_reload_success == 1.0 %}
-      <span style="margin-left:0.4rem;color:var(--green);">&#10003; OK</span>
-    {% else %}
-      <span style="margin-left:0.4rem;color:var(--red);">&#10007; failed</span>
-    {% endif %}
-    {% if global_stat.config_reloads_total is not none %}
-      <span style="color:var(--text-dim);margin-left:0.3rem;">({{ global_stat.config_reloads_total | int }} reloads)</span>
-    {% endif %}
-  </span>
+  {{ stat_tile("Config", "OK" if global_stat.config_last_reload_success == 1.0 else "Reload failed",
+               state="up" if global_stat.config_last_reload_success == 1.0 else "crit",
+               hint=(global_stat.config_reloads_total|int ~ " reloads") if global_stat.config_reloads_total is not none else none) }}
   {% endif %}
-
-  {% if global_stat.process_memory_bytes is not none %}
-  <span>
-    <span style="color:var(--text-muted);">Memory</span>
-    {% set mb = (global_stat.process_memory_bytes / 1048576) %}
-    <span style="margin-left:0.4rem;">
-      {% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.0f"|format(mb) }} MB{% endif %}
-    </span>
-  </span>
+  {% if global_stat.open_conns_total is not none %}{{ stat_tile("Open connections", global_stat.open_conns_total|int) }}{% endif %}
+  {% if global_stat.process_memory_bytes is not none %}{{ stat_tile("Memory", global_stat.process_memory_bytes|bytes) }}{% endif %}
   {% endif %}
+</div>
 
-  <span style="margin-left:auto;color:var(--text-dim);font-size:0.78rem;">
-    {{ global_stat.scraped_at.strftime("%H:%M:%S") }} UTC
-  </span>
+{% macro router_card(r) %}
+<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
+    <div class="host-health-head">
+      {{ status_badge(r.state, "Failing" if r.state == 'crit' else "Warning") }}
+      <h3 class="section-title" style="margin:0;word-break:break-all;">{{ r.name }}</h3>
+    </div>
+    <p class="problem-detail" style="margin-bottom:0.5rem;">{{ r.why }}</p>
+    <table class="table" style="font-size:0.85rem;">
+        <tr><td class="field-hint">Req/s</td><td>{{ "%.2f"|format(r.latest.request_rate) }}</td><td>{{ r.sparkline_req|safe }}</td></tr>
+        <tr><td class="field-hint">Bandwidth</td><td>{{ r.latest.response_bytes_rate|rate }}</td><td></td></tr>
+        <tr><td class="field-hint">p95</td><td class="tone-{{ status_info(status_class(r.latest.latency_p95_ms, 'latency'))[0] }}">{{ "%.0f"|format(r.latest.latency_p95_ms) }} ms</td><td>{{ r.sparkline_p95|safe }}</td></tr>
+        <tr><td class="field-hint">p99</td><td>{{ "%.0f"|format(r.latest.latency_p99_ms) }} ms</td><td></td></tr>
+        <tr><td class="field-hint">4xx</td><td class="{{ 'tone-warn' if r.latest.error_rate_4xx_pct > error_4xx else '' }}">{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%</td><td></td></tr>
+        <tr><td class="field-hint">5xx</td><td class="{{ 'tone-crit' if r.latest.error_rate_5xx_pct > error_5xx else '' }}">{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%</td><td>{{ r.sparkline_5xx|safe }}</td></tr>
+    </table>
+    <div class="field-hint" style="margin-top:0.5rem;">Last scraped {{ r.latest.scraped_at|ago }}</div>
+</div>
+{% endmacro %}
+
+{% if problems or healthy or gone or certs or old_certs %}
+{# ── Routers that need attention ─────────────────────────────────────────── #}
+{% if problems %}
+<div style="column-width:340px;column-gap:1rem;">
+  {% for r in problems %}{{ router_card(r) }}{% endfor %}
 </div>
 {% endif %}
 
-{# ── Per-service cards + cert card in shared columns layout ───────────────── #}
-{% if router_data or certs %}
-<div style="column-width:340px;column-count:3;column-gap:1rem;">
-
-{% for r in router_data %}
-<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
-    <h3 class="section-title" style="margin-bottom:0.75rem;word-break:break-all;">{{ r.name }}</h3>
-    <table class="table" style="font-size:0.85rem;">
-        <tr>
-            <td style="color:var(--text-muted);padding:0.2rem 0;">Req/s</td>
-            <td style="padding:0.2rem 0.5rem;">{{ "%.2f"|format(r.latest.request_rate) }}</td>
-            <td style="padding:0.2rem 0;">{{ r.sparkline_req | safe }}</td>
-        </tr>
-        <tr>
-            <td style="color:var(--text-muted);padding:0.2rem 0;">Bandwidth</td>
-            <td style="padding:0.2rem 0.5rem;">
-              {% set bps = r.latest.response_bytes_rate %}
-              {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
-              {% elif bps >= 1024 %}{{ "%.1f"|format(bps / 1024) }} KB/s
-              {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
-            </td>
-            <td style="padding:0.2rem 0;"></td>
-        </tr>
-        <tr>
-            <td style="color:var(--text-muted);padding:0.2rem 0;">p95 ms</td>
-            <td style="padding:0.2rem 0.5rem;">{{ "%.1f"|format(r.latest.latency_p95_ms) }}</td>
-            <td style="padding:0.2rem 0;">{{ r.sparkline_p95 | safe }}</td>
-        </tr>
-        <tr>
-            <td style="color:var(--text-muted);padding:0.2rem 0;">p99 ms</td>
-            <td style="padding:0.2rem 0.5rem;">{{ "%.1f"|format(r.latest.latency_p99_ms) }}</td>
-            <td style="padding:0.2rem 0;"></td>
-        </tr>
-        <tr>
-            <td style="color:var(--text-muted);padding:0.2rem 0;">4xx %</td>
-            <td style="color:{% if r.latest.error_rate_4xx_pct > 1 %}var(--yellow){% else %}var(--green){% endif %};padding:0.2rem 0.5rem;">
-                {{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%
-            </td>
-            <td style="padding:0.2rem 0;"></td>
-        </tr>
-        <tr>
-            <td style="color:var(--text-muted);padding:0.2rem 0;">5xx %</td>
-            <td style="color:{% if r.latest.error_rate_5xx_pct > 0.1 %}var(--red){% else %}var(--green){% endif %};padding:0.2rem 0.5rem;">
-                {{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%
-            </td>
-            <td style="padding:0.2rem 0;">{{ r.sparkline_5xx | safe }}</td>
-        </tr>
-    </table>
-    <div style="color:var(--text-dim);font-size:0.75rem;margin-top:0.5rem;">
-        Last scraped {{ r.latest.scraped_at.strftime("%H:%M:%S") }} UTC
-    </div>
-</div>
-{% endfor %}
-
-{% if certs %}
-<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
-  <div class="section-title" style="margin-bottom:0.75rem;">TLS Certificates</div>
-  <table class="table" style="font-size:0.83rem;width:100%;">
-    {% for cert in certs %}
+{# ── Healthy routers, folded ─────────────────────────────────────────────── #}
+{% if healthy %}
+<details class="card" id="traefik-healthy" style="margin-bottom:1rem;">
+  <summary class="section-title" style="cursor:pointer;">{{ healthy|length }} healthy router{{ 's' if healthy|length != 1 }}</summary>
+  <div class="table-wrap" style="margin-top:0.75rem;">
+  <table class="table" style="font-size:0.85rem;">
+    <thead><tr><th>Router</th><th>Req/s</th><th>Bandwidth</th><th>p95</th><th>4xx</th><th>5xx</th></tr></thead>
+    <tbody>
+    {% for r in healthy %}
     <tr>
-      <td style="padding:0.2rem 0 0.2rem 0;word-break:break-all;">{{ cert.cn }}</td>
-      <td style="text-align:right;padding:0.2rem 0;white-space:nowrap;font-weight:600;
-        {% if cert.days_remaining < 7 %}color:var(--red);
-        {% elif cert.days_remaining < 30 %}color:var(--yellow);
-        {% else %}color:var(--green);{% endif %}">
-        {{ cert.days_remaining | int }}d
-      </td>
-    </tr>
-    <tr>
-      <td colspan="2" style="padding:0 0 0.4rem 0;color:var(--text-dim);font-size:0.75rem;">
-        {{ cert.not_after.strftime("%Y-%m-%d") }}
-      </td>
+      <td style="word-break:break-all;">{{ r.name }}</td>
+      <td>{{ "%.2f"|format(r.latest.request_rate) }}</td>
+      <td>{{ r.latest.response_bytes_rate|rate }}</td>
+      <td class="tone-{{ status_info(status_class(r.latest.latency_p95_ms, 'latency'))[0] }}">{{ "%.0f"|format(r.latest.latency_p95_ms) }} ms</td>
+      <td>{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%</td>
+      <td>{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%</td>
     </tr>
     {% endfor %}
+    </tbody>
   </table>
+  </div>
+</details>
+{% endif %}
+
+{# ── Routers Traefik no longer reports: their last numbers are history, not a
+   state, so they are listed by when they were last seen and nothing else. ── #}
+{% if gone %}
+<details class="card" style="margin-bottom:1rem;">
+  <summary class="section-title" style="cursor:pointer;">{{ gone|length }} router{{ 's' if gone|length != 1 }} no longer reported</summary>
+  <p class="field-hint" style="margin:0.5rem 0;">Traefik stopped reporting these, usually because the service behind them was removed.</p>
+  <ul class="problem-list">
+    {% for r in gone %}
+    <li><span style="word-break:break-all;">{{ r.name }}</span> <span class="problem-detail">last seen {{ r.latest.scraped_at|ago }}</span></li>
+    {% endfor %}
+  </ul>
+</details>
+{% endif %}
+{% if healthy %}
+<script>
+  // The fragment is re-polled; keep the healthy list as the reader left it.
+  (function () {
+    var el = document.getElementById("traefik-healthy");
+    if (!el) return;
+    el.open = !!window.__traefikHealthyOpen;
+    el.addEventListener("toggle", function () { window.__traefikHealthyOpen = el.open; });
+  })();
+</script>
+{% endif %}
+
+{# ── Certificates Traefik still serves, soonest expiry first; then the ones
+   it no longer reports (renewed or retired), kept here but off the
+   dashboard (#5254). #}
+{% if certs or old_certs %}
+<div class="card">
+  <div class="section-title" style="margin-bottom:0.75rem;">TLS certificates</div>
+  {% if certs %}
+  <table class="table" style="font-size:0.85rem;">
+    <thead><tr><th>Name</th><th>Routers</th><th>Expires</th><th style="text-align:right;">Left</th></tr></thead>
+    <tbody>
+    {% for cert in certs %}
+    <tr>
+      <td style="word-break:break-all;">{{ cert.cn }}{% if cert.sans %}<div class="field-hint">also {{ cert.sans|join(", ") }}</div>{% endif %}</td>
+      <td class="field-hint">{% if cert.routers is none %}—{% elif cert.routers %}{{ cert.routers|join(", ") }}{% else %}not in use{% endif %}</td>
+      <td class="field-hint">{{ cert.not_after.strftime("%Y-%m-%d") }}</td>
+      <td style="text-align:right;white-space:nowrap;" class="tone-{{ status_info(status_class(cert.days_remaining, 'tls_days'))[0] }}">{{ cert_left(cert.days_remaining) }}</td>
+    </tr>
+    {% endfor %}
+    </tbody>
+  </table>
+  {% endif %}
+  {% if old_certs %}
+  <details style="margin-top:0.75rem;">
+    <summary class="field-hint">No longer served ({{ old_certs|length }}) — renewed or retired; Traefik stopped reporting them</summary>
+    <table class="table" style="font-size:0.85rem;margin-top:0.5rem;">
+      <thead><tr><th>Name</th><th>Expires</th></tr></thead>
+      <tbody>
+      {% for cert in old_certs %}
+      <tr style="color:var(--text-muted);">
+        <td style="word-break:break-all;">{{ cert.cn }}</td>
+        <td class="field-hint">{{ cert.not_after.strftime("%Y-%m-%d") }}</td>
+      </tr>
+      {% endfor %}
+      </tbody>
+    </table>
+  </details>
+  {% endif %}
 </div>
 {% endif %}
 
-</div>
 {% else %}
 <div class="card">
-    <p style="color:#606080;">No Traefik metrics yet. Configure <code>metrics_url</code> in <code>config.yaml</code> under <code>plugins.traefik</code>.</p>
+    <p class="field-hint">No Traefik metrics yet. Configure <code>metrics_url</code> in <code>config.yaml</code> under <code>plugins.traefik</code>.</p>
 </div>
 {% endif %}

plugins/traefik/templates/traefik/traffic.html

@@ -2,26 +2,25 @@
 {% if summary %}
 
 {# ── Overview bar ─────────────────────────────────────────────────────────── #}
-<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--card-bg);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;align-items:center;">
+<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--bg-card);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;align-items:center;">
   <span>
     <span style="color:var(--text-muted);">Total</span>
-    <span style="margin-left:0.4rem;font-weight:600;">{{ summary.total }}</span>
+    <span style="margin-left:0.4rem;font-weight:500;">{{ summary.total }}</span>
   </span>
   <span>
     <span style="color:var(--text-muted);">Internal</span>
-    <span style="margin-left:0.4rem;color:var(--green);font-weight:600;">{{ summary.internal }}</span>
+    <span style="margin-left:0.4rem;color:var(--green);font-weight:500;">{{ summary.internal }}</span>
   </span>
   <span>
     <span style="color:var(--text-muted);">External</span>
-    <span style="margin-left:0.4rem;color:{% if summary.external_pct > 50 %}var(--yellow){% else %}var(--green){% endif %};font-weight:600;">
+    <span style="margin-left:0.4rem;color:{% if summary.external_pct > 50 %}var(--yellow){% else %}var(--green){% endif %};font-weight:500;">
       {{ summary.external }} ({{ "%.1f"|format(summary.external_pct) }}%)
     </span>
   </span>
   <span>
     <span style="color:var(--text-muted);">Bytes</span>
     <span style="margin-left:0.4rem;">
-      {% set mb = summary.total_bytes / 1048576 %}
-      {% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.1f"|format(mb) }} MB{% endif %}
+      {{ summary.total_bytes|bytes }}
     </span>
   </span>
 
@@ -93,9 +92,13 @@
 
 {% else %}
 <div class="card" style="color:var(--text-muted);font-size:0.85rem;">
-  No access log data yet.
   {% if not access_log_enabled %}
-  Enable <code>access_log.enabled: true</code> in your plugin config and bind-mount the Traefik log directory.
+  The access log is off. Turn it on in <a href="/settings/plugins/traefik/">Settings → Plugins → Traefik</a>,
+  and mount Traefik's log directory into Steward so it can read the file.
+  {% else %}
+  No requests in the last {{ range_key }}. The access log is on; if this stays empty, check that
+  {% if access_log_path %}<code>{{ access_log_path }}</code>{% else %}the log path{% endif %} is mounted into Steward
+  and that Traefik is writing to it.
   {% endif %}
 </div>
 {% endif %}

plugins/traefik/templates/traefik/widget.html

@@ -1,56 +1,55 @@
-{# plugins/traefik/templates/traefik/widget.html #}
-{% if expiring_certs %}
-<div style="margin-bottom:0.6rem;">
-{% for cert in expiring_certs %}
-<div style="display:flex;justify-content:space-between;font-size:0.8rem;padding:0.15rem 0;
-  {% if cert.days_remaining < 7 %}color:var(--red);{% else %}color:var(--yellow);{% endif %}">
-  <span>&#9888; {{ cert.cn }}</span>
-  <span>{{ cert.days_remaining | int }}d</span>
-</div>
-{% endfor %}
-</div>
-{% endif %}
-
-{% if router_data %}
-{% for r in router_data %}
-<div class="ping-row">
-  <div class="ping-meta">
-    <span class="ping-name" style="font-size:0.8rem;word-break:break-all;">{{ r.name }}</span>
-  </div>
-  <div style="flex:1;display:flex;gap:1.5rem;font-size:0.82rem;font-variant-numeric:tabular-nums;">
-    <span title="Requests/s">
-      <span style="color:var(--text-muted);">req/s</span>
-      <span style="margin-left:0.3rem;">{{ "%.2f"|format(r.latest.request_rate) }}</span>
-    </span>
-    <span title="Bandwidth">
-      <span style="color:var(--text-muted);">bw</span>
-      <span style="margin-left:0.3rem;">
-        {% set bps = r.latest.response_bytes_rate %}
-        {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
-        {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
-        {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
-      </span>
-    </span>
-    <span title="p95 latency">
-      <span style="color:var(--text-muted);">p95</span>
-      <span style="margin-left:0.3rem;
-        {% if r.latest.latency_p95_ms > 500 %}color:var(--red){% elif r.latest.latency_p95_ms > 200 %}color:var(--yellow){% else %}color:var(--green){% endif %};">
-        {{ "%.0f"|format(r.latest.latency_p95_ms) }}ms
-      </span>
-    </span>
-    {% if r.latest.error_rate_5xx_pct > 0 %}
-    <span title="5xx error rate">
-      <span style="color:var(--red);">{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% 5xx</span>
-    </span>
-    {% endif %}
-  </div>
-</div>
-{% endfor %}
-{% if total_routers > 5 %}
-<div style="color:var(--text-dim);font-size:0.75rem;padding:0.25rem 0;">
-  +{{ total_routers - 5 }} more — <a href="/plugins/traefik/">view all →</a>
-</div>
-{% endif %}
+{# plugins/traefik/templates/traefik/widget.html — the dashboard Routers widget
+   (#5254): a verdict, routers in trouble with why, then the busiest healthy
+   ones, under one header. Only routers Traefik still reports. A certificate
+   shows on the router it serves (TLS column, days left); when Traefik's API
+   can't say which hosts a router serves, the expiring certificates are listed
+   after the routers instead. #}
+{% from "_macros.html" import status_dot %}
+{% if not total %}
+<div class="w-verdict" style="color:var(--text-muted);">No Traefik data yet.</div>
 {% else %}
-<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No Traefik data yet.</p>
+{% set crit = problems | selectattr("state", "equalto", "crit") | list %}
+<div class="w-verdict {{ 'tone-crit' if crit else ('tone-warn' if problems or loose_certs else 'tone-ok') }}">
+  {%- if problems|length == 1 %}{{ problems[0].name.split('@')[0] }}: {{ problems[0].why }}.
+  {%- elif problems %}{{ problems|length }} of {{ total }} routers need a look.
+  {%- else %}All {{ total }} router{{ 's' if total != 1 }} healthy{% if loose_certs %}; {{ loose_certs|length }} certificate{{ 's' if loose_certs|length != 1 }} expiring{% endif %}.{% endif -%}
+</div>
+<table class="w-table">
+  <colgroup><col><col style="width:4rem"><col class="w-wide-only" style="width:5.5rem"><col style="width:4.2rem">{% if domains_known %}<col style="width:4.5rem">{% endif %}</colgroup>
+  <thead><tr><th>Router</th><th>Req/s</th><th class="w-wide-only">Bandwidth</th><th>p95</th>{% if domains_known %}<th>TLS</th>{% endif %}</tr></thead>
+  <tbody>
+  {% for r in problems + healthy %}
+  {% set m = r.latest %}
+  <tr>
+    <td class="w-name" title="{{ r.name }}">
+      {{- status_dot(r.state) -}}
+      {{ r.name.split('@')[0] }}
+      {%- if r.why %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}</span>{% endif %}
+    </td>
+    <td>{% if m.request_rate is none %}—{% else %}{{ "%.2f"|format(m.request_rate) }}{% endif %}</td>
+    <td class="w-wide-only">{% if m.response_bytes_rate is none %}—{% else %}{{ m.response_bytes_rate|rate }}{% endif %}</td>
+    <td style="{{ threshold_style(m.latency_p95_ms, 'latency') if m.latency_p95_ms is not none }}">
+      {%- if m.latency_p95_ms is none %}—{% else %}{{ "%.0f"|format(m.latency_p95_ms) }} ms{% endif -%}
+    </td>
+    {% if domains_known %}
+    <td {% if r.cert %}style="{{ threshold_style(r.cert.days, 'tls_days') }}" title="Certificate {{ r.cert.cn }}"{% endif %}>
+      {%- if r.cert %}{{ cert_left(r.cert.days) }}{% else %}—{% endif -%}
+    </td>
+    {% endif %}
+  </tr>
+  {% endfor %}
+  </tbody>
+</table>
+{% if healthy_more %}<div class="w-more">+{{ healthy_more }} more healthy{% if session.user_id %} · <a href="/plugins/traefik/">all routers</a>{% endif %}</div>{% endif %}
+{% if loose_certs %}
+<div class="w-section">Certificates expiring</div>
+{% for c in loose_certs %}
+<div class="w-item">
+  {{- status_dot(status_class(c.days, 'tls_days')) -}}
+  <span class="w-label" title="{{ c.cn }}">{{ c.cn }}</span>
+  <span class="w-meta" style="{{ threshold_style(c.days, 'tls_days') }}">{{ cert_left(c.days) }}</span>
+</div>
+{% endfor %}
+<div class="w-note">Traefik's API didn't say which hosts each router serves, so these can't be shown on their routers.</div>
+{% endif %}
 {% endif %}

plugins/traefik/templates/traefik/widget_access_log.html

@@ -5,11 +5,11 @@
 
 <div style="display:flex;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
   <div>
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;">{{ summary.total | int }}</span>
+    <span style="font-size:1.4rem;font-weight:500;line-height:1;">{{ summary.total | int }}</span>
     <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">total req</span>
   </div>
   <div>
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--orange);">
+    <span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--orange);">
       {{ "%.1f" | format(summary.external_pct) }}%
     </span>
     <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">external</span>

plugins/traefik/templates/traefik/widget_request_chart.html

@@ -1,75 +1,100 @@
-{# traefik/widget_request_chart.html — Chart.js request rate + error % over time #}
+{# traefik/widget_request_chart.html — requests per second across all routers
+   and the share answered 5xx, over the last few hours (#5256). The verdict
+   carries the figures, so the widget still says something if the chart
+   can't draw; a chart that fails says why in place of the canvas. #}
 {% if not labels_json or labels_json == '[]' %}
-<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No data for last {{ hours }}h.</div>
+<div class="w-verdict" style="color:var(--text-muted);">No Traefik requests recorded in the last {{ hours }}h.</div>
 {% else %}
-<div style="position:relative;height:160px;">
-  <canvas id="chart-traefik-req-{{ widget_id }}"></canvas>
+<div class="w-verdict {{ 'tone-crit' if peak_err and peak_err >= 1 else ('tone-warn' if peak_err else 'tone-ok') }}">
+  Now {{ "%.2f"|format(now_req) }} req/s across all routers, peak {{ "%.2f"|format(peak_req) }}.
+  {%- if peak_err %} 5xx peaked at {{ "%.2f"|format(peak_err) }}% of responses.{% else %} No 5xx.{% endif %}
+</div>
+<div style="position:relative;height:160px;" id="chart-traefik-req-box-{{ widget_id }}">
+  <canvas id="chart-traefik-req-{{ widget_id }}" aria-label="Requests per second and 5xx share, last {{ hours }} hours"></canvas>
 </div>
 <script>
 (function() {
+  var box = document.getElementById("chart-traefik-req-box-{{ widget_id }}");
   var ctx = document.getElementById("chart-traefik-req-{{ widget_id }}");
-  if (!ctx || !window.Chart) return;
-  var existing = Chart.getChart(ctx);
-  if (existing) existing.destroy();
-  new Chart(ctx.getContext("2d"), {
-    type: "line",
-    data: {
-      labels: {{ labels_json | safe }},
-      datasets: [
-        {
-          label: "Req/s",
-          data: {{ req_rate_json | safe }},
-          borderColor: "#7c5ef4",
-          backgroundColor: "rgba(124,94,244,0.08)",
-          fill: true,
-          tension: 0.3,
-          pointRadius: 0,
-          borderWidth: 1.5,
-          yAxisID: "y",
-        },
-        {
-          label: "5xx %",
-          data: {{ error_rate_json | safe }},
-          borderColor: "#e83848",
-          fill: false,
-          tension: 0.3,
-          pointRadius: 0,
-          borderWidth: 1.5,
-          yAxisID: "y2",
-        },
-      ]
-    },
-    options: {
-      responsive: true,
-      maintainAspectRatio: false,
-      animation: false,
-      plugins: {
-        legend: {
-          labels: { color: "#6858a0", boxWidth: 10, font: { size: 10 } }
-        }
+  if (!ctx) return;
+  function fail(why) {
+    box.style.height = "auto";
+    box.innerHTML = '<div class="w-note"></div>';
+    box.firstChild.textContent = "The chart couldn't be drawn: " + why;
+  }
+  if (!window.Chart) { fail("the chart library didn't load."); return; }
+  try {
+    var existing = Chart.getChart(ctx);
+    if (existing) existing.destroy();
+    var times = {{ labels_json | safe }};
+    var hhmm = function (ms) {
+      return new Date(ms).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" });
+    };
+    var chart = new Chart(ctx.getContext("2d"), {
+      type: "line",
+      data: {
+        labels: times,
+        datasets: [
+          {
+            label: "Requests/s (all routers)",
+            data: {{ req_rate_json | safe }},
+            borderColor: stewardColor("--blue"),
+            backgroundColor: stewardColor("--blue-dim"),
+            fill: true, tension: 0.3, pointRadius: 0, borderWidth: 1.5, yAxisID: "y",
+          },
+          {
+            label: "5xx share %",
+            data: {{ error_rate_json | safe }},
+            borderColor: stewardColor("--red"),
+            fill: false, tension: 0.3, pointRadius: 0, borderWidth: 1.5, yAxisID: "y2",
+          },
+        ]
       },
-      scales: {
-        x: { display: false },
-        y: {
-          min: 0,
-          grid: { color: "rgba(30,30,62,0.8)" },
-          ticks: { color: "#6858a0", font: { size: 10 } },
-          title: { display: true, text: "req/s", color: "#6858a0", font: { size: 9 } },
-        },
-        y2: {
-          position: "right",
-          min: 0,
-          max: 100,
-          grid: { drawOnChartArea: false },
-          ticks: { color: "#e83848", font: { size: 10 } },
-          title: { display: true, text: "5xx%", color: "#e83848", font: { size: 9 } },
+      options: {
+        responsive: true,
+        maintainAspectRatio: false,
+        animation: false,
+        interaction: { mode: "index", intersect: false },
+        plugins: {
+          legend: { labels: { boxWidth: 10, font: { size: 10 } } },
+          tooltip: { callbacks: { title: function (items) { return items.length ? hhmm(times[items[0].dataIndex]) : ""; } } },
         },
+        scales: {
+          x: {
+            ticks: {
+              font: { size: 10 }, maxRotation: 0, autoSkip: true, maxTicksLimit: 6,
+              callback: function (value) { return hhmm(times[value]); },
+            },
+            grid: { display: false },
+          },
+          y: {
+            min: 0,
+            ticks: { font: { size: 10 } },
+            title: { display: true, text: "req/s", font: { size: 9 } },
+          },
+          y2: {
+            position: "right",
+            min: 0,
+            suggestedMax: 5,
+            grid: { drawOnChartArea: false },
+            ticks: { color: stewardColor("--red"), font: { size: 10 } },
+            title: { display: true, text: "5xx %", color: stewardColor("--red"), font: { size: 9 } },
+          },
+        }
       }
-    }
-  });
+    });
+    // The chart is built while the dashboard may still be placing this cell,
+    // and a chart sized then can stay blank until a window resize (#5214).
+    // Measure and draw again once layout has settled.
+    setTimeout(function () {
+      if (!ctx.isConnected) return;
+      chart.resize();
+      chart.update("none");
+    }, 250);
+  } catch (err) {
+    fail(err && err.message ? err.message : String(err));
+  }
 })();
 </script>
-<div style="font-size:0.72rem;color:var(--text-muted);margin-top:0.35rem;text-align:right;">
-  last {{ hours }}h
-</div>
+<div class="w-note" style="text-align:right;">last {{ hours }}h, your local time</div>
 {% endif %}

plugins/unifi/__init__.py

@@ -16,6 +16,14 @@ def setup(app: "Quart") -> None:
         UnifiDevice, UnifiKnownClient, UnifiEvent, UnifiAlarm,
         UnifiSpeedtestResult, UnifiNetwork, UnifiPortForward, UnifiFwRule,
     )
+    from steward.core.cleanup import prune_by_age
+    # Samples and history older than data.retention_days are pruned by the
+    # cleanup (#5155). Devices, known clients and config tables are current state.
+    for column in (UnifiWanStat.scraped_at, UnifiClientSnapshot.scraped_at,
+                   UnifiWlanStat.scraped_at, UnifiDpiSnapshot.scraped_at,
+                   UnifiEvent.event_time, UnifiAlarm.alarm_time,
+                   UnifiSpeedtestResult.run_at):
+        prune_by_age(column)
 
 
 def get_scheduled_tasks() -> list:
@@ -31,3 +39,9 @@ def get_blueprint():
 def get_nav() -> list[dict]:
     # Surfaces in the sidebar's Infrastructure group (UniFi network overview).
     return [{"label": "UniFi", "href": "/plugins/unifi/"}]
+
+
+def get_alert_metrics() -> dict[str, list[str]]:
+    """What scheduler.py records: the WAN (resource "wan") and the client count
+    (resource "clients")."""
+    return {"unifi": ["is_up", "latency_ms", "total_clients"]}

plugins/unifi/models.py

@@ -1,10 +1,10 @@
 # plugins/unifi/models.py
 from __future__ import annotations
-import uuid
 from datetime import datetime, timezone
 from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text
 from sqlalchemy.orm import Mapped, mapped_column
 from steward.models.base import Base
+from steward.core.ids import time_ordered_id
 
 
 # ── Time-series (one row per scrape) ──────────────────────────────────────────
@@ -13,7 +13,7 @@ class UnifiWanStat(Base):
     """WAN interface health — one row per scrape."""
     __tablename__ = "unifi_wan_stats"
 
-    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
     scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
     is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     wan_ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
@@ -27,7 +27,7 @@ class UnifiClientSnapshot(Base):
     """Aggregated client counts + top talkers — one row per scrape."""
     __tablename__ = "unifi_client_snapshots"
 
-    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
     scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
     total_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     wireless_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
@@ -41,7 +41,7 @@ class UnifiWlanStat(Base):
     """Per-SSID/radio stats — one row per scrape per SSID+radio combination."""
     __tablename__ = "unifi_wlan_stats"
 
-    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
     scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
     ssid: Mapped[str] = mapped_column(String(255), nullable=False)
     radio: Mapped[str | None] = mapped_column(String(8), nullable=True)    # ng=2.4GHz, na=5GHz, 6e=6GHz
@@ -56,7 +56,7 @@ class UnifiDpiSnapshot(Base):
     """DPI application/category traffic snapshot — one row per scrape (JSON blob)."""
     __tablename__ = "unifi_dpi_snapshots"
 
-    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
     scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
     # JSON: [{"cat_id", "cat_name", "rx_bytes", "tx_bytes"}] aggregated across all clients
     by_category_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")

plugins/unifi/routes.py

@@ -324,33 +324,3 @@ async def widget_clients():
         snapshot=snapshot,
         widget_id=widget_id,
     )
-
-
-@unifi_bp.get("/widget/devices")
-@require_role(UserRole.viewer)
-async def widget_devices():
-    """HTMX dashboard widget: infrastructure devices with optional type filter."""
-    type_filter = request.args.get("type_filter", "all")
-    widget_id = request.args.get("wid", "0")
-
-    async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name)
-        )
-        all_devices = result.scalars().all()
-
-    if type_filter == "wireless":
-        devices = [d for d in all_devices if d.device_type == "uap"]
-    elif type_filter == "wired":
-        devices = [d for d in all_devices if d.device_type != "uap"]
-    elif type_filter == "offline":
-        devices = [d for d in all_devices if d.state != 1]
-    else:
-        devices = list(all_devices)
-
-    return await render_template(
-        "unifi/widget_devices.html",
-        devices=devices,
-        type_filter=type_filter,
-        widget_id=widget_id,
-    )

plugins/unifi/scheduler.py

@@ -54,7 +54,7 @@ async def _do_poll(app: "Quart") -> None:
 
     from .client import UnifiClient
     from .models import UnifiClientSnapshot, UnifiDevice, UnifiWanStat
-    from steward.core.alerts import record_metric
+    from steward.core.alerts import MetricBatch
 
     cfg = app.config["PLUGINS"]["unifi"]
 
@@ -171,28 +171,27 @@ async def _do_poll(app: "Quart") -> None:
                     ))
 
             # ── Alert metrics ─────────────────────────────────────────────────
+            batch = MetricBatch()
             if wan_stat.latency_ms is not None:
-                await record_metric(
-                    session=session,
+                batch.add(
                     source_module="unifi",
                     resource_name="wan",
                     metric_name="latency_ms",
                     value=wan_stat.latency_ms,
                 )
-            await record_metric(
-                session=session,
+            batch.add(
                 source_module="unifi",
                 resource_name="wan",
                 metric_name="is_up",
                 value=1.0 if wan_stat.is_up else 0.0,
             )
-            await record_metric(
-                session=session,
+            batch.add(
                 source_module="unifi",
                 resource_name="clients",
                 metric_name="total_clients",
                 value=float(len(clients)),
             )
+            await batch.record(session)
 
     logger.debug(
         "UniFi poll: wan=%s, clients=%d, devices=%d",

plugins/unifi/templates/unifi/dpi.html

@@ -11,7 +11,7 @@
   <div class="card" style="break-inside:avoid;margin-bottom:1rem;">
     <div class="section-title" style="margin-bottom:0.75rem;">
       Traffic by Category
-      <span style="float:right;font-weight:normal;font-size:0.78rem;color:var(--text-dim);">{{ snapshot.scraped_at.strftime("%H:%M") }}</span>
+      <span style="float:right;font-weight:normal;font-size:0.78rem;color:var(--text-dim);">as of {{ snapshot.scraped_at|ago }}</span>
     </div>
     {% set total_bytes = categories | sum(attribute='rx_bytes') + categories | sum(attribute='tx_bytes') %}
     <table class="table" style="width:100%;font-size:0.82rem;">
@@ -21,10 +21,7 @@
       <tr>
         <td style="padding:0.25rem 0.5rem 0.25rem 0;">{{ cat.cat_name or cat.cat_id }}</td>
         <td style="padding:0.25rem 0.3rem;text-align:right;color:var(--text-muted);font-size:0.78rem;white-space:nowrap;">
-          {% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB
-          {% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB
-          {% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB
-          {% else %}{{ bytes }} B{% endif %}
+          {{ bytes|bytes }}
         </td>
         <td style="padding:0.25rem 0;width:4rem;">
           <div style="background:var(--border);border-radius:2px;height:6px;overflow:hidden;">
@@ -50,10 +47,7 @@
         </td>
         <td style="padding:0.25rem 0;text-align:right;font-size:0.78rem;color:var(--text-muted);white-space:nowrap;">
           {% set bytes = c.bytes %}
-          {% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB
-          {% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB
-          {% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB
-          {% else %}{{ bytes }} B{% endif %}
+          {{ bytes|bytes }}
         </td>
       </tr>
       {% endfor %}

plugins/unifi/templates/unifi/events.html

@@ -23,18 +23,12 @@
       <th style="text-align:left;font-weight:normal;padding:0 0 0.4rem;">Message</th>
     </tr>
     {% for ev in events %}
-    <tr style="{% if loop.index is divisibleby 2 %}background:var(--row-alt,rgba(255,255,255,0.02));{% endif %}">
+    <tr style="{% if loop.index is divisibleby 2 %}background:var(--fs-table-stripe);{% endif %}">
       <td style="padding:0.25rem 0.5rem 0.25rem 0;color:var(--text-dim);font-size:0.77rem;white-space:nowrap;">
-        {{ ev.event_time.strftime("%Y-%m-%d %H:%M:%S") }}
+        {{ ev.event_time|ago }}
       </td>
       <td style="padding:0.25rem 0.5rem;">
-        <span style="font-size:0.75rem;padding:0.1rem 0.4rem;border-radius:3px;background:
-          {% if ev.category == 'wlan' %}rgba(96,96,192,0.25)
-          {% elif ev.category == 'wan' %}rgba(96,160,96,0.25)
-          {% elif ev.category == 'ids' %}rgba(192,64,64,0.25)
-          {% else %}rgba(128,128,128,0.15){% endif %};">
-          {{ ev.category }}
-        </span>
+        <span class="badge badge-{{ {'wlan': 'info', 'wan': 'ok', 'ids': 'crit'}.get(ev.category, 'dim') }}">{{ ev.category }}</span>
       </td>
       <td style="padding:0.25rem 0.5rem;color:var(--text-dim);font-family:monospace;font-size:0.75rem;">
         {{ ev.event_key.replace("EVT_", "") if ev.event_key else "" }}

plugins/unifi/templates/unifi/inventory.html

@@ -61,7 +61,7 @@
       {% if rule.ruleset != ns.current_rs %}
       {% set ns.current_rs = rule.ruleset %}
       <tr>
-        <td colspan="3" style="padding:0.4rem 0 0.2rem;color:var(--text-muted);font-size:0.75rem;font-weight:600;letter-spacing:0.04em;text-transform:uppercase;">
+        <td colspan="3" style="padding:0.4rem 0 0.2rem;color:var(--text-muted);font-size:0.75rem;font-weight:500;letter-spacing:0.04em;text-transform:uppercase;">
           {{ rule.ruleset }}
         </td>
       </tr>
@@ -73,12 +73,7 @@
           </span>
         </td>
         <td style="padding:0.2rem 0.3rem;white-space:nowrap;">
-          <span style="font-size:0.75rem;padding:0.1rem 0.35rem;border-radius:3px;background:
-            {% if rule.action == 'accept' %}rgba(64,160,64,0.2)
-            {% elif rule.action == 'drop' %}rgba(192,64,64,0.2)
-            {% else %}rgba(192,128,64,0.2){% endif %};">
-            {{ rule.action }}
-          </span>
+          <span class="badge badge-{{ {'accept': 'ok', 'drop': 'crit'}.get(rule.action, 'warn') }}">{{ rule.action }}</span>
         </td>
         <td style="padding:0.2rem 0;color:var(--text-dim);font-size:0.76rem;">
           {% if rule.protocol %}{{ rule.protocol }}{% endif %}
@@ -122,7 +117,7 @@
           {{ kc.ip or "—" }}
         </td>
         <td style="padding:0.2rem 0;text-align:right;color:var(--text-dim);font-size:0.75rem;white-space:nowrap;">
-          {% if kc.last_seen %}{{ kc.last_seen.strftime("%m-%d %H:%M") }}{% else %}—{% endif %}
+          {% if kc.last_seen %}{{ kc.last_seen|ago }}{% else %}—{% endif %}
         </td>
       </tr>
       {% endfor %}

plugins/unifi/templates/unifi/rows.html

@@ -1,57 +1,20 @@
 {# plugins/unifi/templates/unifi/rows.html #}
 
-{# ── WAN health bar ───────────────────────────────────────────────────────── #}
+{% from "_macros.html" import stat_tile, status_badge, status_dot %}
+{# ── WAN health ───────────────────────────────────────────────────────────── #}
 {% if latest_wan %}
-<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--card-bg);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;align-items:center;">
-  <span>
-    <span style="color:var(--text-muted);">WAN</span>
-    <span style="margin-left:0.4rem;font-weight:600;color:{% if latest_wan.is_up %}var(--green){% else %}var(--red){% endif %};">
-      {% if latest_wan.is_up %}&#10003; Up{% else %}&#10007; Down{% endif %}
-    </span>
-    {% if latest_wan.wan_ip %}
-    <span style="color:var(--text-dim);margin-left:0.4rem;font-family:monospace;font-size:0.8rem;">{{ latest_wan.wan_ip }}</span>
-    {% endif %}
-  </span>
-
+<div class="stat-strip">
+  {{ stat_tile("WAN", "Up" if latest_wan.is_up else "Down", state="up" if latest_wan.is_up else "down",
+               hint=latest_wan.wan_ip) }}
   {% if latest_wan.latency_ms is not none %}
-  <span>
-    <span style="color:var(--text-muted);">Latency</span>
-    <span style="margin-left:0.4rem;
-      {% if latest_wan.latency_ms > 100 %}color:var(--red){% elif latest_wan.latency_ms > 50 %}color:var(--yellow){% else %}color:var(--green){% endif %};">
-      {{ "%.1f"|format(latest_wan.latency_ms) }}ms
-    </span>
-    <span style="margin-left:0.3rem;">{{ sparkline_latency | safe }}</span>
-  </span>
+  {{ stat_tile("Latency", "%.0f ms"|format(latest_wan.latency_ms), state=status_class(latest_wan.latency_ms, 'latency'),
+               hint=sparkline_latency|safe) }}
   {% endif %}
-
   {% if latest_wan.rx_bytes_rate is not none %}
-  <span>
-    <span style="color:var(--text-muted);">&#8595; RX</span>
-    <span style="margin-left:0.3rem;">
-      {% set bps = latest_wan.rx_bytes_rate %}
-      {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
-      {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s
-      {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
-    </span>
-    <span style="margin-left:0.3rem;">{{ sparkline_rx | safe }}</span>
-  </span>
-  <span>
-    <span style="color:var(--text-muted);">&#8593; TX</span>
-    <span style="margin-left:0.3rem;">
-      {% set bps = latest_wan.tx_bytes_rate %}
-      {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
-      {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s
-      {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
-    </span>
-    <span style="margin-left:0.3rem;">{{ sparkline_tx | safe }}</span>
-  </span>
-  {% endif %}
-
-  {% if latest_wan.uptime_seconds is not none %}
-  <span style="color:var(--text-dim);font-size:0.8rem;">
-    up {{ (latest_wan.uptime_seconds // 3600) }}h {{ ((latest_wan.uptime_seconds % 3600) // 60) }}m
-  </span>
+  {{ stat_tile("Download", latest_wan.rx_bytes_rate|rate, hint=sparkline_rx|safe) }}
+  {{ stat_tile("Upload", latest_wan.tx_bytes_rate|rate, hint=sparkline_tx|safe) }}
   {% endif %}
+  {% if latest_wan.uptime_seconds is not none %}{{ stat_tile("WAN uptime", latest_wan.uptime_seconds|duration) }}{% endif %}
 </div>
 {% endif %}
 
@@ -67,7 +30,7 @@
     </div>
     <div style="display:flex;gap:1.5rem;font-size:0.9rem;margin-bottom:0.75rem;font-variant-numeric:tabular-nums;">
       <span>
-        <span style="font-size:1.4rem;font-weight:700;color:var(--text);">{{ latest_clients.total_clients }}</span>
+        <span style="font-size:1.4rem;color:var(--text);">{{ latest_clients.total_clients }}</span>
         <span style="color:var(--text-muted);font-size:0.8rem;margin-left:0.2rem;">total</span>
       </span>
       <span style="color:var(--text-muted);font-size:0.85rem;align-self:flex-end;">
@@ -82,12 +45,12 @@
   {# Infrastructure devices card #}
   {% if devices %}
   <div class="card" style="break-inside:avoid;margin-bottom:1rem;">
-    <div class="section-title" style="margin-bottom:0.75rem;">Infrastructure Devices</div>
+    <div class="section-title" style="margin-bottom:0.75rem;">Infrastructure devices</div>
     <table class="table" style="font-size:0.83rem;width:100%;">
       {% for d in devices %}
       <tr>
         <td style="padding:0.2rem 0.5rem 0.2rem 0;">
-          <span style="color:{% if d.state == 1 %}var(--green){% else %}var(--red){% endif %};">&#9679;</span>
+          {{ status_dot('up' if d.state == 1 else 'down') }}
           <span style="margin-left:0.3rem;">{{ d.name or d.mac }}</span>
         </td>
         <td style="padding:0.2rem 0.3rem;color:var(--text-muted);font-size:0.78rem;">{{ d.model or d.device_type or "" }}</td>
@@ -96,7 +59,7 @@
       {% if d.state == 1 and d.uptime_seconds %}
       <tr>
         <td colspan="3" style="padding:0 0 0.3rem 1.2rem;color:var(--text-dim);font-size:0.75rem;">
-          up {{ (d.uptime_seconds // 86400) }}d {{ ((d.uptime_seconds % 86400) // 3600) }}h
+          up {{ d.uptime_seconds|duration }}
         </td>
       </tr>
       {% endif %}
@@ -108,7 +71,7 @@
   {# Top talkers card #}
   {% if top_clients %}
   <div class="card" style="break-inside:avoid;margin-bottom:1rem;">
-    <div class="section-title" style="margin-bottom:0.75rem;">Top Talkers</div>
+    <div class="section-title" style="margin-bottom:0.75rem;">Top talkers</div>
     <table class="table" style="font-size:0.82rem;width:100%;">
       {% for c in top_clients %}
       <tr>
@@ -121,13 +84,9 @@
         <td style="text-align:right;padding:0.2rem 0;white-space:nowrap;font-size:0.78rem;">
           {% set rx = c.rx_rate %}{% set tx = c.tx_rate %}
           <span style="color:var(--text-muted);">&#8595;</span>
-          {% if rx >= 1048576 %}{{ "%.1f"|format(rx/1048576) }}M
-          {% elif rx >= 1024 %}{{ "%.0f"|format(rx/1024) }}K
-          {% else %}{{ "%.0f"|format(rx) }}B{% endif %}
+          {{ rx|rate }}
           <span style="color:var(--text-muted);margin-left:0.3rem;">&#8593;</span>
-          {% if tx >= 1048576 %}{{ "%.1f"|format(tx/1048576) }}M
-          {% elif tx >= 1024 %}{{ "%.0f"|format(tx/1024) }}K
-          {% else %}{{ "%.0f"|format(tx) }}B{% endif %}
+          {{ tx|rate }}
         </td>
       </tr>
       {% endfor %}
@@ -138,7 +97,7 @@
   {# WLAN SSIDs card #}
   {% if wlan_latest %}
   <div class="card" style="break-inside:avoid;margin-bottom:1rem;">
-    <div class="section-title" style="margin-bottom:0.75rem;">Wireless Networks</div>
+    <div class="section-title" style="margin-bottom:0.75rem;">Wireless networks</div>
     <table class="table" style="font-size:0.83rem;width:100%;">
       <tr style="color:var(--text-muted);font-size:0.78rem;">
         <th style="text-align:left;font-weight:normal;padding:0 0.5rem 0.3rem 0;">SSID</th>
@@ -149,10 +108,7 @@
       <tr>
         <td style="padding:0.2rem 0.5rem 0.2rem 0;">{{ w.ssid }}</td>
         <td style="text-align:right;padding:0.2rem 0.3rem;">{{ w.num_sta }}</td>
-        <td style="text-align:right;padding:0.2rem 0;
-          {% if w.satisfaction is not none %}
-            {% if w.satisfaction >= 80 %}color:var(--green){% elif w.satisfaction >= 60 %}color:var(--yellow){% else %}color:var(--red){% endif %}
-          {% endif %};">
+        <td style="text-align:right;padding:0.2rem 0;" class="{% if w.satisfaction is not none %}tone-{{ 'ok' if w.satisfaction >= 80 else 'warn' if w.satisfaction >= 60 else 'crit' }}{% endif %}">
           {% if w.satisfaction is not none %}{{ w.satisfaction }}%{% else %}—{% endif %}
         </td>
       </tr>
@@ -164,19 +120,19 @@
   {# Speedtest card #}
   {% if speedtest %}
   <div class="card" style="break-inside:avoid;margin-bottom:1rem;">
-    <div class="section-title" style="margin-bottom:0.75rem;">Last Speed Test</div>
+    <div class="section-title" style="margin-bottom:0.75rem;">Last speed test</div>
     <div style="display:flex;gap:1.5rem;font-size:0.88rem;font-variant-numeric:tabular-nums;">
       {% if speedtest.download_mbps is not none %}
       <span>
         <span style="color:var(--text-muted);font-size:0.8rem;">&#8595; DL</span>
-        <span style="font-weight:600;margin-left:0.3rem;">{{ "%.1f"|format(speedtest.download_mbps) }}</span>
+        <span style="margin-left:0.3rem;">{{ "%.1f"|format(speedtest.download_mbps) }}</span>
         <span style="color:var(--text-muted);font-size:0.78rem;">Mbps</span>
       </span>
       {% endif %}
       {% if speedtest.upload_mbps is not none %}
       <span>
         <span style="color:var(--text-muted);font-size:0.8rem;">&#8593; UL</span>
-        <span style="font-weight:600;margin-left:0.3rem;">{{ "%.1f"|format(speedtest.upload_mbps) }}</span>
+        <span style="margin-left:0.3rem;">{{ "%.1f"|format(speedtest.upload_mbps) }}</span>
         <span style="color:var(--text-muted);font-size:0.78rem;">Mbps</span>
       </span>
       {% endif %}
@@ -190,7 +146,7 @@
     {% if speedtest.server_name %}
     <div style="margin-top:0.4rem;color:var(--text-dim);font-size:0.78rem;">via {{ speedtest.server_name }}</div>
     {% endif %}
-    <div style="margin-top:0.25rem;color:var(--text-dim);font-size:0.75rem;">{{ speedtest.run_at.strftime("%Y-%m-%d %H:%M") }} UTC</div>
+    <div style="margin-top:0.25rem;color:var(--text-dim);font-size:0.75rem;">{{ speedtest.run_at|ago }}</div>
   </div>
   {% endif %}
 

plugins/unifi/templates/unifi/security.html

@@ -6,7 +6,7 @@
     <div class="section-title" style="margin-bottom:0.75rem;">
       Active Alarms
       {% if active_alarms %}
-      <span style="float:right;background:var(--red);color:#fff;font-size:0.72rem;padding:0.1rem 0.45rem;border-radius:10px;font-weight:600;">{{ active_alarms|length }}</span>
+      <span class="count-badge" style="float:right;">{{ active_alarms|length }}</span>
       {% endif %}
     </div>
     {% if not active_alarms %}
@@ -21,7 +21,7 @@
           </div>
           <div>{{ al.message }}</div>
           <div style="color:var(--text-dim);font-size:0.75rem;margin-top:0.1rem;">
-            {{ al.alarm_time.strftime("%Y-%m-%d %H:%M") }} UTC
+            {{ al.alarm_time|ago }}
           </div>
         </td>
       </tr>
@@ -45,7 +45,7 @@
           <div style="color:var(--text-dim);font-size:0.8rem;">{{ al.message }}</div>
         </td>
         <td style="padding:0.25rem 0;text-align:right;color:var(--text-dim);font-size:0.75rem;white-space:nowrap;">
-          {{ al.alarm_time.strftime("%m-%d %H:%M") }}
+          {{ al.alarm_time|ago }}
         </td>
       </tr>
       {% endfor %}

plugins/unifi/templates/unifi/widget.html

@@ -37,16 +37,12 @@
     <span>
       <span style="color:var(--text-muted);">&#8595;</span>
       {% set bps = latest_wan.rx_bytes_rate %}
-      {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
-      {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
-      {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
+      {{ bps|rate }}
     </span>
     <span>
       <span style="color:var(--text-muted);">&#8593;</span>
       {% set bps = latest_wan.tx_bytes_rate %}
-      {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
-      {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
-      {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
+      {{ bps|rate }}
     </span>
     {% endif %}
   </div>
@@ -59,7 +55,7 @@
     <span class="ping-name" style="font-size:0.8rem;">Clients</span>
   </div>
   <div style="flex:1;display:flex;gap:1rem;font-size:0.82rem;">
-    <span style="font-weight:600;">{{ latest_clients.total_clients }}</span>
+    <span style="font-weight:500;">{{ latest_clients.total_clients }}</span>
     <span style="color:var(--text-muted);">{{ latest_clients.wireless_clients }}w / {{ latest_clients.wired_clients }}e</span>
   </div>
 </div>
@@ -76,14 +72,10 @@
       {% set total_bps = c.tx_rate + c.rx_rate %}
       <span style="color:var(--text-muted);">&#8595;</span>
       {% set rx = c.rx_rate %}
-      {% if rx >= 1048576 %}{{ "%.1f"|format(rx / 1048576) }}M
-      {% elif rx >= 1024 %}{{ "%.0f"|format(rx / 1024) }}K
-      {% else %}{{ "%.0f"|format(rx) }}B{% endif %}
+      {{ rx|rate }}
       <span style="color:var(--text-muted);margin-left:0.4rem;">&#8593;</span>
       {% set tx = c.tx_rate %}
-      {% if tx >= 1048576 %}{{ "%.1f"|format(tx / 1048576) }}M
-      {% elif tx >= 1024 %}{{ "%.0f"|format(tx / 1024) }}K
-      {% else %}{{ "%.0f"|format(tx) }}B{% endif %}
+      {{ tx|rate }}
     </div>
   </div>
   {% endfor %}

plugins/unifi/templates/unifi/widget_clients.html

@@ -5,15 +5,15 @@
 
 <div style="display:flex;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
   <div>
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;">{{ snapshot.total_clients }}</span>
+    <span style="font-size:1.4rem;font-weight:500;line-height:1;">{{ snapshot.total_clients }}</span>
     <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">total</span>
   </div>
   <div>
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--accent);">{{ snapshot.wireless_clients }}</span>
+    <span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--accent);">{{ snapshot.wireless_clients }}</span>
     <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">wireless</span>
   </div>
   <div>
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--text-muted);">{{ snapshot.wired_clients }}</span>
+    <span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--text-muted);">{{ snapshot.wired_clients }}</span>
     <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">wired</span>
   </div>
 </div>

plugins/unifi/templates/unifi/widget_devices.html

@@ -1,36 +0,0 @@
-{# unifi/widget_devices.html — dashboard widget: infrastructure devices #}
-{% if not devices %}
-<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">
-  {% if type_filter != "all" %}No {{ type_filter }} devices found.{% else %}No devices found.{% endif %}
-</div>
-{% else %}
-
-{% set online = devices | selectattr("state", "equalto", 1) | list %}
-{% set offline = devices | rejectattr("state", "equalto", 1) | list %}
-
-<div style="display:flex;gap:1rem;margin-bottom:0.75rem;">
-  <div>
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--green);">{{ online | length }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">online</span>
-  </div>
-  {% if offline %}
-  <div>
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--red);">{{ offline | length }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">offline</span>
-  </div>
-  {% endif %}
-</div>
-
-<div style="display:grid;gap:3px;">
-  {% for d in devices %}
-  <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
-    <span class="dot {% if d.state == 1 %}dot-up{% else %}dot-down{% endif %}"></span>
-    <span style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
-      {{ d.name or d.mac }}
-    </span>
-    <span style="font-size:0.72rem;color:var(--text-dim);flex-shrink:0;">{{ d.model or d.device_type or "" }}</span>
-  </div>
-  {% endfor %}
-</div>
-
-{% endif %}

pyproject.toml

@@ -18,6 +18,9 @@ dependencies = [
     "click>=8.0",
     "httpx>=0.27",
     "cryptography>=42",
+    # Floor, not a cap: MCPServer (mcp.server.mcpserver) and the mcp.types
+    # mirror that steward/mcp/ is written against first ship in SDK 2.2.
+    "mcp>=2.2",
 ]
 
 [project.optional-dependencies]

steward/ai_access/routes.py

@@ -0,0 +1,108 @@
+"""AI access: personal API tokens for MCP clients.
+
+Every logged-in user manages their own tokens here, whatever their role — a
+token acts as its user, capped at the role it was given. These routes are
+session-only: a share-link viewer has no user to mint tokens for, and no token
+can reach them (MCP tools never mint or revoke tokens).
+"""
+from __future__ import annotations
+from datetime import datetime, timezone
+
+from quart import Blueprint, abort, current_app, redirect, render_template, request
+from sqlalchemy import select
+
+from steward.auth.middleware import current_user_id, require_role
+from steward.core import api_tokens
+from steward.core.audit import log_audit
+from steward.core.settings import public_base_url
+from steward.mcp.server import MCP_PATH
+from steward.models.users import User, UserRole
+
+ai_access_bp = Blueprint("ai_access", __name__, url_prefix="/ai-access")
+
+_EXPIRY_CHOICES = [("", "Never"), ("30", "30 days"), ("90", "90 days"), ("365", "1 year")]
+
+
+def _mcp_url() -> str:
+    """The /mcp endpoint as a client outside this request reaches it."""
+    return public_base_url(request) + MCP_PATH
+
+
+async def _session_user(db) -> User:
+    user_id = current_user_id()
+    if not user_id:
+        abort(403)
+    user = (await db.execute(select(User).where(User.id == user_id))).scalar_one_or_none()
+    if user is None or not user.is_active:
+        abort(403)
+    return user
+
+
+async def _render_index(db, user: User, *, error: str | None = None, status: int = 200):
+    tokens = await api_tokens.list_tokens(db, user.id)
+    now = datetime.now(timezone.utc)
+    rows = [{"token": t, "state": api_tokens.token_state(t, now)} for t in tokens]
+    body = await render_template(
+        "ai_access/index.html",
+        rows=rows,
+        user_role=user.role,
+        grantable=api_tokens.grantable_roles(user.role),
+        expiry_choices=_EXPIRY_CHOICES,
+        error=error,
+        mcp_url=_mcp_url(),
+    )
+    return body, status
+
+
+@ai_access_bp.get("/")
+@require_role(UserRole.viewer)
+async def index():
+    async with current_app.db_sessionmaker() as db:
+        user = await _session_user(db)
+        return await _render_index(db, user)
+
+
+@ai_access_bp.post("/tokens")
+@require_role(UserRole.viewer)
+async def create():
+    form = await request.form
+    name = form.get("name", "")
+    expires = form.get("expires_in_days", "")
+    try:
+        role = UserRole(form.get("role", UserRole.viewer.value))
+    except ValueError:
+        role = None
+
+    async with current_app.db_sessionmaker() as db:
+        user = await _session_user(db)
+        if role is None:
+            return await _render_index(db, user, error="Pick a role for the token.", status=400)
+        try:
+            raw, token = await api_tokens.create_token(
+                db, user_id=user.id, user_role=user.role, name=name, role=role,
+                expires_in_days=int(expires) if expires.isdigit() and int(expires) > 0 else None,
+            )
+        except ValueError as exc:
+            return await _render_index(db, user, error=str(exc), status=400)
+        await db.commit()
+
+    await log_audit(current_app, user.id, user.username, "api_token.create",
+                    entity_type="api_token", entity_id=str(token.id),
+                    detail={"name": token.name, "role": token.role.value})
+    return await render_template("ai_access/created.html", raw_token=raw, token=token,
+                                 mcp_url=_mcp_url())
+
+
+@ai_access_bp.post("/tokens/<int:token_id>/revoke")
+@require_role(UserRole.viewer)
+async def revoke(token_id: int):
+    async with current_app.db_sessionmaker() as db:
+        user = await _session_user(db)
+        token = await api_tokens.revoke_token(db, user_id=user.id, token_id=token_id)
+        await db.commit()
+    if token is None:
+        abort(404)
+    await log_audit(current_app, user.id, user.username, "api_token.revoke",
+                    entity_type="api_token", entity_id=str(token_id),
+                    detail={"name": token.name})
+    return redirect("/ai-access/")

steward/alerts/routes.py

@@ -4,64 +4,47 @@ from quart import Blueprint, render_template, request, redirect, url_for, curren
 from steward.core.audit import log_audit
 from sqlalchemy import select
 from steward.auth.middleware import require_role
+from steward.alerts import service as alert_svc
+from steward.core.paging import PAGE_SIZE, next_page
+from steward.core.time_range import parse_range
 from steward.ansible import sources as src_module
-from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertOperator, MaintenanceWindow
-from steward.models.metrics import PluginMetric
+from steward.models.alerts import AlertRule, AlertOperator, AlertState, MaintenanceWindow
 from steward.models.users import UserRole
 
 alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
 
 _OPERATORS = [(op.value, op.value) for op in AlertOperator]
 
-# Static catalog: source_module → available metric names.
-# Unified monitors emit metrics under their TYPE (icmp/tcp/dns/http), each with
-# is_up (1.0 up / 0.0 down) and response_ms (where the probe times it).
-METRIC_CATALOG: dict[str, list[str]] = {
-    "icmp":    ["is_up", "response_ms"],
-    "tcp":     ["is_up", "response_ms"],
-    "dns":     ["is_up", "response_ms"],
-    "http":    ["is_up", "response_ms"],
-    "traefik": ["request_rate", "error_rate", "latency_p50_ms", "latency_p95_ms",
-                "latency_p99_ms", "response_bytes_rate", "cert_expiry_days"],
-    "unifi":   ["is_up", "latency_ms", "total_clients"],
-    # restart_count = cumulative restarts (alert on crash-looping); is_healthy =
-    # 1.0 healthy / 0.0 unhealthy from the container HEALTHCHECK (alert on <1).
-    "docker":  ["cpu_pct", "mem_pct", "restart_count", "is_healthy"],
-    "host_agent": [
-        "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
-        "disk_used_pct_worst", "load_1m", "load_5m", "load_15m", "uptime_secs",
-    ],
-    # snmp metrics are user-defined OID labels — populated dynamically from plugin_metrics
-    "snmp":    [],
-}
-
-# Ordered list for source module picker
-_SOURCE_MODULES = list(METRIC_CATALOG.keys())
+def _form_condition(form) -> dict:
+    """The operator and threshold the form asks for. An up/down metric's form
+    sends the word it fires on (binary_when) instead: down is < 0.5, up is
+    >= 0.5, the midpoint of the 1.0 / 0.0 such a metric records."""
+    when = form.get("binary_when")
+    if when in ("down", "up") and form.get("metric_name") in ("is_up", "is_healthy"):
+        return {"operator": "<" if when == "down" else ">=", "threshold": "0.5"}
+    return {"operator": form.get("operator", ""), "threshold": form.get("threshold")}
 
 
-async def _get_resources(db, source_module: str) -> list[str]:
-    """Return sorted distinct resource names for a source module.
+def _rule_form(form) -> dict:
+    """The rule form's fields, as alert_svc.rule_fields takes them."""
+    return {
+        "name": form.get("name", ""),
+        "source_module": form.get("source_module", ""),
+        "resource_name": form.get("resource_name", ""),
+        "metric_name": form.get("metric_name", ""),
+        **_form_condition(form),
+        "consecutive_failures_required": form.get("consecutive_failures_required"),
+        "consecutive_ok_required": form.get("consecutive_ok_required"),
+        "notify_no_data": "notify_no_data" in form,
+    }
 
-    For monitor types (icmp/tcp/dns/http), supplements with monitor names so
-    the list is populated even before any metrics have been recorded.
-    """
-    result = await db.execute(
-        select(PluginMetric.resource_name)
-        .where(PluginMetric.source_module == source_module)
-        .distinct()
-        .order_by(PluginMetric.resource_name)
-    )
-    resources: set[str] = {r for (r,) in result}
 
-    if source_module in ("icmp", "tcp", "dns", "http"):
-        from steward.models.monitors import Monitor
-        mon_result = await db.execute(
-            select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name)
-        )
-        for (name,) in mon_result:
-            resources.add(name)
-
-    return sorted(resources)
+def _form_time(value) -> datetime:
+    """A datetime-local field, read as UTC."""
+    try:
+        return datetime.fromisoformat((value or "").strip()).replace(tzinfo=timezone.utc)
+    except ValueError:
+        raise ValueError(f"{value!r} is not a date and time.") from None
 
 
 def _is_admin() -> bool:
@@ -123,22 +106,78 @@ def _parse_ansible_action(form) -> tuple[dict | None, str | None]:
 @require_role(UserRole.viewer)
 async def list_alerts():
     async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(AlertState, AlertRule)
-            .join(AlertRule, AlertState.rule_id == AlertRule.id)
-            .where(AlertState.state.in_([
-                AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending
-            ]))
-            .order_by(AlertState.last_transition_at.desc())
-        )
-        active = result.all()
+        active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
+        rules = await alert_svc.list_rules(db)
+        states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
 
-        rules_result = await db.execute(
-            select(AlertRule).order_by(AlertRule.name)
-        )
-        rules = rules_result.scalars().all()
+    # Every rule with what it is doing now; disabled rules last.
+    rules.sort(key=lambda r: (not r.enabled, r.name.lower()))
+    rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states}
+    return await render_template(
+        "alerts/list.html", active=active, rules=rules, rule_state=rule_state,
+        operators=_OPERATORS, poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60))
 
-    return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)
+
+@alerts_bp.get("/active")
+@require_role(UserRole.viewer)
+async def active_fragment():
+    """The Active section alone, which the alerts page polls."""
+    async with current_app.db_sessionmaker() as db:
+        active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
+    return await render_template("alerts/_active.html", active=active)
+
+
+def _history_row(event, rule) -> dict:
+    """One transition as the history page shows it."""
+    from steward.core.alerts import MUTED, RELEASED
+    from steward.core.formatting import metric_label, metric_value
+    notes, muted_by, released = [], None, False
+    for part in (event.note or "").split("; "):
+        if part.startswith(MUTED):
+            muted_by = part[len(MUTED):].strip() or "a maintenance window"
+        elif part == RELEASED:
+            released = True
+        elif part:
+            notes.append(part)
+    metric = rule.metric_name
+    if event.metric_value is None:
+        value = f"No {metric_label(metric)} reading"
+    elif metric in ("is_up", "is_healthy"):
+        value = metric_value(metric, event.metric_value).capitalize()
+    else:
+        value = f"{metric_label(metric)} {metric_value(metric, event.metric_value)}"
+    return {
+        "at": event.transitioned_at, "rule": rule,
+        "from_state": event.from_state, "to_state": event.to_state,
+        "value": value, "condition": alert_svc.rule_condition(rule),
+        "notes": notes, "muted_by": muted_by, "released": released,
+        "deliveries": [(label, sent, error) for label, sent, error in (
+            ("email", event.email_sent, event.email_error),
+            ("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
+        "ansible_run_id": event.ansible_run_id,
+    }
+
+
+@alerts_bp.get("/history")
+@require_role(UserRole.viewer)
+async def alert_history():
+    """Every alert transition in a range, newest first; one rule's with ?rule=.
+    A page at a time as the reader scrolls: ?before=<cursor> is the next
+    page's rows alone (core.paging)."""
+    since, range_key = parse_range(request.args.get("range") or "7d")
+    before = request.args.get("before")
+    async with current_app.db_sessionmaker() as db:
+        rules = await alert_svc.list_rules(db)
+        chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
+        rows = await alert_svc.history(db, since, chosen.id if chosen else None, PAGE_SIZE, before)
+    rows, after = next_page(rows, lambda row: (row[0].transitioned_at, row[0].id))
+    ctx = {"events": [_history_row(e, r) for e, r in rows], "after": after,
+           "rules": rules, "chosen": chosen, "current_range": range_key}
+    if before:
+        return await render_template("alerts/_history_page.html", **ctx)
+    if request.headers.get("HX-Request"):
+        return await render_template("alerts/_history_rows.html", **ctx)
+    return await render_template("alerts/history.html", **ctx)
 
 
 @alerts_bp.get("/api/rule_fields")
@@ -148,18 +187,9 @@ async def rule_fields_api():
     current_resource = request.args.get("current_resource", "")
     current_metric = request.args.get("current_metric", "")
     async with current_app.db_sessionmaker() as db:
-        resources = await _get_resources(db, source) if source else []
-    metrics = METRIC_CATALOG.get(source, [])
-    # Dynamic sources (e.g. snmp) have empty static catalog — query live metric names
-    if source and not metrics:
-        async with current_app.db_sessionmaker() as db:
-            m_result = await db.execute(
-                select(PluginMetric.metric_name)
-                .where(PluginMetric.source_module == source)
-                .distinct()
-                .order_by(PluginMetric.metric_name)
-            )
-            metrics = [r for (r,) in m_result]
+        resources = await alert_svc.resources_for(db, source) if source else []
+    async with current_app.db_sessionmaker() as db:
+        metrics = await alert_svc.metrics_for(db, source)
     return await render_template(
         "alerts/_rule_fields.html",
         resources=resources,
@@ -172,17 +202,18 @@ async def rule_fields_api():
 @alerts_bp.get("/rules/new")
 @require_role(UserRole.operator)
 async def new_rule():
-    first_source = _SOURCE_MODULES[0]
+    first_source = alert_svc.source_modules()[0]
     async with current_app.db_sessionmaker() as db:
-        resources = await _get_resources(db, first_source)
+        resources = await alert_svc.resources_for(db, first_source)
     return await render_template(
         "alerts/rules_form.html",
         rule=None,
         operators=_OPERATORS,
-        source_modules=_SOURCE_MODULES,
+        source_modules=alert_svc.source_modules(),
         initial_source=first_source,
         resources=resources,
-        metrics=METRIC_CATALOG.get(first_source, []),
+        metrics=alert_svc.metric_catalog().get(first_source, []),
+        catalog=alert_svc.metric_catalog(),
         ansible_sources=_ansible_source_names(),
     )
 
@@ -195,15 +226,16 @@ async def edit_rule(rule_id: str):
         rule = result.scalar_one_or_none()
         if rule is None:
             return "Not found", 404
-        resources = await _get_resources(db, rule.source_module)
+        resources = await alert_svc.resources_for(db, rule.source_module)
     return await render_template(
         "alerts/rules_form.html",
         rule=rule,
         operators=_OPERATORS,
-        source_modules=_SOURCE_MODULES,
+        source_modules=alert_svc.source_modules(),
         initial_source=rule.source_module,
         resources=resources,
-        metrics=METRIC_CATALOG.get(rule.source_module, []),
+        metrics=alert_svc.metric_catalog().get(rule.source_module, []),
+        catalog=alert_svc.metric_catalog(),
         ansible_sources=_ansible_source_names(),
     )
 
@@ -213,36 +245,20 @@ async def edit_rule(rule_id: str):
 async def create_rule():
     form = await request.form
     user_id = session.get("user_id")
-    now = datetime.now(timezone.utc)
     # Ansible action is admin-only; non-admins simply can't set one.
     ansible_action = None
     if _is_admin():
         ansible_action, err = _parse_ansible_action(form)
         if err:
             return err, 400
-    rule = AlertRule(
-        name=form["name"].strip(),
-        source_module=form["source_module"].strip(),
-        resource_name=form["resource_name"].strip(),
-        metric_name=form["metric_name"].strip(),
-        operator=AlertOperator(form["operator"]),
-        threshold=float(form["threshold"]),
-        consecutive_failures_required=int(form.get("consecutive_failures_required") or 1),
-        enabled=True,
-        ansible_action=ansible_action,
-        created_by=user_id,
-    )
-    state = AlertState(
-        rule_id=rule.id,
-        state=AlertStateEnum.inactive,
-        last_transition_at=now,
-    )
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
-            db.add(rule)
-            await db.flush()
-            state.rule_id = rule.id
-            db.add(state)
+            try:
+                fields = await alert_svc.rule_fields(db, **_rule_form(form))
+            except ValueError as exc:
+                return str(exc), 400
+            rule = await alert_svc.add_rule(db, fields, created_by=user_id,
+                                            ansible_action=ansible_action)
     await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created",
                     entity_type="alert_rule", entity_id=rule.name,
                     detail={"source": rule.source_module, "resource": rule.resource_name,
@@ -267,13 +283,11 @@ async def update_rule(rule_id: str):
             rule = result.scalar_one_or_none()
             if rule is None:
                 return "Not found", 404
-            rule.name = form["name"].strip()
-            rule.source_module = form["source_module"].strip()
-            rule.resource_name = form["resource_name"].strip()
-            rule.metric_name = form["metric_name"].strip()
-            rule.operator = AlertOperator(form["operator"])
-            rule.threshold = float(form["threshold"])
-            rule.consecutive_failures_required = int(form.get("consecutive_failures_required") or 1)
+            try:
+                fields = await alert_svc.rule_fields(db, **_rule_form(form))
+            except ValueError as exc:
+                return str(exc), 400
+            await alert_svc.apply_rule_change(db, rule, fields)
             if apply_action:
                 rule.ansible_action = ansible_action
     await log_audit(current_app, session.get("user_id"), session.get("username", ""),
@@ -290,7 +304,7 @@ async def toggle_rule(rule_id: str):
             rule = result.scalar_one_or_none()
             if rule is None:
                 return "Not found", 404
-            rule.enabled = not rule.enabled
+            await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled)
     action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
     await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                     action, entity_type="alert_rule", entity_id=rule.name)
@@ -319,15 +333,17 @@ async def delete_rule(rule_id: str):
 async def list_maintenance():
     now = datetime.now(timezone.utc)
     async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
-        )
-        windows = result.scalars().all()
+        windows = await alert_svc.list_maintenance_windows(db)
+    # Active first (ending soonest first), then upcoming (starting soonest
+    # first); expired ones are history, newest first, folded away.
+    active = sorted((w for w in windows if w.start_at <= now <= w.end_at), key=lambda w: w.end_at)
+    upcoming = sorted((w for w in windows if w.start_at > now), key=lambda w: w.start_at)
+    expired = sorted((w for w in windows if w.end_at < now), key=lambda w: w.end_at, reverse=True)
     return await render_template(
         "alerts/maintenance.html",
-        windows=windows,
+        windows=windows, active=active, upcoming=upcoming, expired=expired,
         now=now,
-        source_modules=_SOURCE_MODULES,
+        source_modules=alert_svc.source_modules(),
     )
 
 
@@ -337,7 +353,7 @@ async def new_maintenance():
     return await render_template(
         "alerts/maintenance_form.html",
         window=None,
-        source_modules=_SOURCE_MODULES,
+        source_modules=alert_svc.source_modules(),
     )
 
 
@@ -346,26 +362,18 @@ async def new_maintenance():
 async def create_maintenance():
     form = await request.form
     user_id = session.get("user_id")
-    start_at = datetime.fromisoformat(form["start_at"]).replace(tzinfo=timezone.utc)
-    end_at = datetime.fromisoformat(form["end_at"]).replace(tzinfo=timezone.utc)
-    scope_source = form.get("scope_source", "").strip() or None
-    scope_resource = form.get("scope_resource", "").strip() or None
-    if scope_source is None:
-        scope_resource = None  # can't have resource without source
-    window = MaintenanceWindow(
-        name=form["name"].strip(),
-        start_at=start_at,
-        end_at=end_at,
-        scope_source=scope_source,
-        scope_resource=scope_resource,
-        created_by=user_id,
-    )
+    try:
+        fields = alert_svc.window_fields(
+            name=form.get("name", ""),
+            start_at=_form_time(form.get("start_at")), end_at=_form_time(form.get("end_at")),
+            scope_source=form.get("scope_source"), scope_resource=form.get("scope_resource"))
+    except ValueError as exc:
+        return str(exc), 400
+    window = MaintenanceWindow(created_by=user_id, **fields)
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
             db.add(window)
-    scope_desc = "all" if not scope_source else (
-        f"{scope_source}/{scope_resource}" if scope_resource else f"{scope_source}/*"
-    )
+    scope_desc = alert_svc.window_scope(window)
     await log_audit(current_app, user_id, session.get("username", ""),
                     "maintenance_window.created", entity_type="maintenance_window",
                     entity_id=window.name, detail={"scope": scope_desc})
@@ -396,19 +404,12 @@ async def delete_maintenance(window_id: str):
 @require_role(UserRole.operator)
 async def acknowledge(rule_id: str):
     user_id = session.get("user_id")
-    now = datetime.now(timezone.utc)
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
-            result = await db.execute(
-                select(AlertState).where(AlertState.rule_id == rule_id)
-            )
-            state = result.scalar_one_or_none()
-            if state is None:
+            try:
+                await alert_svc.acknowledge(db, rule_id, user_id)
+            except LookupError:
                 return "Not found", 404
-            if state.state != AlertStateEnum.firing:
-                return "Conflict: alert is not in FIRING state", 409
-            state.state = AlertStateEnum.acknowledged
-            state.acknowledged_by = user_id
-            state.acknowledged_at = now
-            state.last_transition_at = now
+            except ValueError as exc:
+                return f"Conflict: {exc}", 409
     return redirect(url_for("alerts.list_alerts"))

steward/alerts/service.py

@@ -0,0 +1,456 @@
+"""Alert queries and validation shared by the alerts pages and the MCP tools.
+
+Each function takes an open session (or plain values), with no Quart request
+context, so the routes and the /mcp tools (which run outside Quart's
+pipeline) call the same code and refuse the same input for the same reasons.
+"""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+
+from sqlalchemy import func, or_, select
+
+from steward.models.alerts import (
+    AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
+)
+from steward.models.metrics import PluginMetric, PluginMetricLatest
+from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
+
+logger = logging.getLogger(__name__)
+
+# What core itself records. Unified monitors emit metrics under their TYPE
+# (icmp/tcp/dns/http), each with is_up (1.0 up / 0.0 down) and response_ms
+# (where the probe times it); "steward" is the app watching itself.
+CORE_METRICS: dict[str, list[str]] = {
+    "icmp":    ["is_up", "response_ms"],
+    "tcp":     ["is_up", "response_ms"],
+    "dns":     ["is_up", "response_ms"],
+    "http":    ["is_up", "response_ms"],
+    "steward": ["open_fds", "open_fds_pct"],
+}
+
+
+def metric_catalog() -> dict[str, list[str]]:
+    """Every source an alert rule can watch, and its metric names.
+
+    Core's own sources, then each loaded plugin's get_alert_metrics() export
+    ({source: [metric names]}), so the catalog is written by whatever emits
+    the metrics and a disabled plugin's sources drop out. An empty list means
+    the names are dynamic (snmp: the OID labels recorded so far).
+    """
+    from steward.core.plugin_manager import get_loaded_plugin_modules
+    catalog = {src: list(names) for src, names in CORE_METRICS.items()}
+    for name, module in get_loaded_plugin_modules().items():
+        hook = getattr(module, "get_alert_metrics", None)
+        if hook is None:
+            continue
+        try:
+            contributed = hook() or {}
+        except Exception:
+            logger.exception("Plugin %r: get_alert_metrics() raised; its metrics are skipped", name)
+            continue
+        for src, names in contributed.items():
+            catalog[str(src)] = [str(n) for n in names]
+    return catalog
+
+
+def source_modules() -> list[str]:
+    """The sources in picker order: core's first, then plugins'."""
+    return list(metric_catalog())
+
+
+# States that need someone's attention: pending (breaching, not yet for long
+# enough), firing, and acknowledged-but-still-breaching.
+ACTIVE_STATES = (AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending)
+
+
+async def active_alerts(db) -> list:
+    """(AlertState, AlertRule) rows that need attention, latest transition
+    first: not yet resolved, or with a series that has gone silent (no_data)."""
+    return list((await db.execute(
+        select(AlertState, AlertRule)
+        .join(AlertRule, AlertState.rule_id == AlertRule.id)
+        .where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
+        .order_by(AlertState.last_transition_at.desc())
+    )).all())
+
+
+def alert_label(state: AlertState) -> str:
+    """What an alert is, in one word: its state, or "no_data" for a quiet rule
+    whose series has gone silent (a firing alert that goes silent stays
+    "firing"; check no_data_since for the mark)."""
+    if state.no_data_since is not None and state.state == AlertStateEnum.inactive:
+        return "no_data"
+    return state.state.value
+
+
+async def latest_value(db, rule: AlertRule) -> float | None:
+    """The newest sample of the series a rule watches, or None before the first
+    (a primary-key read of the latest-value table, #5208)."""
+    return (await db.execute(
+        select(PluginMetricLatest.value)
+        .where(PluginMetricLatest.source_module == rule.source_module,
+               PluginMetricLatest.resource_name == rule.resource_name,
+               PluginMetricLatest.metric_name == rule.metric_name)
+    )).scalar_one_or_none()
+
+
+# Sources whose resources have no page of their own: the plugin's page lists them.
+PLUGIN_PAGES = {"traefik": "/plugins/traefik/", "unifi": "/plugins/unifi/",
+                "snmp": "/plugins/snmp/"}
+
+
+async def resource_url(db, rule: AlertRule) -> str | None:
+    """Where the thing a rule watches is shown, or None when nothing shows it.
+
+    host_agent resources are the host's name, or "<host>:<mount|nic|core>";
+    docker's are "<host>/<container>"; a monitor's is its name under its type.
+    """
+    from steward.models.hosts import Host
+    source, resource = rule.source_module, rule.resource_name
+    if source == "host_agent":
+        name = resource.split(":", 1)[0]
+        host_id = (await db.execute(
+            select(Host.id).where(Host.name == name).limit(1))).scalar_one_or_none()
+        return f"/hosts/{host_id}" if host_id else None
+    if source in MONITOR_TYPE_VALUES:
+        monitor_id = (await db.execute(
+            select(Monitor.id).where(Monitor.type == source, Monitor.name == resource).limit(1)
+        )).scalar_one_or_none()
+        return f"/monitors/{monitor_id}" if monitor_id else None
+    if source == "docker" and "/" in resource:
+        host_name, container = resource.split("/", 1)
+        host_id = (await db.execute(
+            select(Host.id).where(Host.name == host_name).limit(1))).scalar_one_or_none()
+        return f"/plugins/docker/container/{host_id}/{container}" if host_id else None
+    return PLUGIN_PAGES.get(source)
+
+
+# Metrics whose value is a word: the sentence says the word, not the condition.
+_BINARY = ("is_up", "is_healthy")
+
+
+def rule_condition(rule: AlertRule) -> str:
+    """What a rule fires on, in words: "CPU > 90%", "Fullest disk >= 85%",
+    or "is down" for an up/down metric rather than "is_up < 0.5"."""
+    from steward.core.alerts import _is_breached
+    from steward.core.formatting import metric_label, metric_value
+    metric = rule.metric_name
+    if metric in _BINARY:
+        fires_on = [metric_value(metric, v) for v in (1.0, 0.0)
+                    if _is_breached(v, rule.operator, rule.threshold)]
+        if len(fires_on) == 1:
+            return f"is {fires_on[0]}"
+    return f"{metric_label(metric)} {rule.operator.value} {metric_value(metric, rule.threshold)}"
+
+
+def alert_sentence(rule: AlertRule, label: str, value: float | None,
+             since: datetime | None, now: datetime) -> dict:
+    """An alert as one sentence around its resource: {"lead", "resource",
+    "tail"}, read lead + resource + tail, e.g. "CPU on " + "web-1" +
+    " is 94% (> 90%), firing for 12m". label is alert_label(state); since is
+    when that label began (no_data_since for no_data)."""
+    from steward.core.formatting import human_duration, metric_label, metric_value
+    metric = rule.metric_name
+    said = metric_label(metric)
+    held = human_duration((now - since).total_seconds()) if since else None
+    if label == "no_data":
+        return {"lead": f"No {said} reading from ", "resource": rule.resource_name,
+                "tail": f" for {held}" if held else ""}
+    # An acknowledged alert is still breaching: say for how long it has been,
+    # not how long ago someone acknowledged it (the page says that apart).
+    word = "breaching" if label == "acknowledged" else label
+    for_how_long = f", {word} for {held}" if held else f", {word}"
+    if value is None:
+        reading = " has no reading yet"
+    elif metric in _BINARY:
+        return {"lead": "", "resource": rule.resource_name,
+                "tail": f" is {metric_value(metric, value)}{for_how_long}"}
+    else:
+        reading = f" is {metric_value(metric, value)}"
+    condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
+    return {"lead": f"{said} on ", "resource": rule.resource_name,
+            "tail": f"{reading}{condition}{for_how_long}"}
+
+
+# The order the active list reads in: what is breaching and unhandled, then
+# what has gone silent, then what someone has taken, then what may yet clear.
+ACTIVE_ORDER = {"firing": 0, "no_data": 1, "acknowledged": 2, "pending": 3}
+
+_BREACHING = (AlertStateEnum.pending.value, AlertStateEnum.firing.value,
+              AlertStateEnum.acknowledged.value)
+
+
+async def breach_started(db, rule_ids) -> dict[str, datetime]:
+    """{rule_id: when its current breach began}: the newest transition into
+    pending or firing from a quiet state. Acknowledging moves an alert's
+    last_transition_at, so that column cannot say this."""
+    if not rule_ids:
+        return {}
+    return dict((await db.execute(
+        select(AlertEvent.rule_id, func.max(AlertEvent.transitioned_at))
+        .where(AlertEvent.rule_id.in_(list(rule_ids)),
+               AlertEvent.from_state.not_in(_BREACHING),
+               AlertEvent.to_state.in_((AlertStateEnum.pending.value, AlertStateEnum.firing.value)))
+        .group_by(AlertEvent.rule_id)
+    )).all())
+
+
+async def active_rows(db, now: datetime) -> list[dict]:
+    """The alerts page's active list, firing first, then no data, then
+    acknowledged, then pending; within each the longest-running first. Each
+    row: its state, rule, label, sentence, the link to what it watches, when
+    the breach began, and who acknowledged it."""
+    from steward.models.users import User
+    alerts = await active_alerts(db)
+    started = await breach_started(db, {rule.id for state, rule in alerts
+                                        if state.state.value in _BREACHING})
+    ackers = {s.acknowledged_by for s, _ in alerts if s.acknowledged_by}
+    names = dict((await db.execute(
+        select(User.id, User.username).where(User.id.in_(ackers)))).all()) if ackers else {}
+    rows = []
+    for state, rule in alerts:
+        label = alert_label(state)
+        if label == "no_data":
+            since = state.no_data_since
+        elif label == "acknowledged":
+            since = started.get(rule.id, state.last_transition_at)
+        else:
+            since = state.last_transition_at
+        rows.append({
+            "state": state, "rule": rule, "label": label, "since": since,
+            "breach_since": started.get(rule.id) if label != "no_data" else None,
+            "acked_by": names.get(state.acknowledged_by, state.acknowledged_by),
+            "says": alert_sentence(rule, label, await latest_value(db, rule), since, now),
+            "url": await resource_url(db, rule),
+        })
+    epoch = datetime.max.replace(tzinfo=timezone.utc)
+    rows.sort(key=lambda r: (ACTIVE_ORDER.get(r["label"], 9), r["since"] or epoch))
+    return rows
+
+
+def watches_host(rule: AlertRule, host_name: str, monitor_keys: set) -> bool:
+    """Whether a rule watches something on this host: its agent series
+    ("<host>" or "<host>:<part>"), one of its containers ("<host>/<name>"),
+    or one of its monitors ((type, name) in monitor_keys)."""
+    source, resource = rule.source_module, rule.resource_name
+    if source == "host_agent":
+        return resource == host_name or resource.startswith(host_name + ":")
+    if source == "docker":
+        return resource.startswith(host_name + "/")
+    return (source, resource) in monitor_keys
+
+
+async def active_count(db) -> int:
+    """How many alerts need attention: the sidebar's number."""
+    from sqlalchemy import func
+    return (await db.execute(
+        select(func.count()).select_from(AlertState)
+        .where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
+    )).scalar_one()
+
+
+async def list_rules(db) -> list[AlertRule]:
+    return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())
+
+
+async def history(db, since: datetime, rule_id: str | None = None,
+                  limit: int = 200, before: str | None = None) -> list[tuple[AlertEvent, AlertRule]]:
+    """Alert transitions since `since`, newest first, with their rule; one
+    rule's when `rule_id` is given; only those older than the `before` cursor
+    when given (core.paging). Returns up to limit+1 rows so a caller can tell
+    the list was cut."""
+    from steward.core.paging import keyset
+    stmt = (select(AlertEvent, AlertRule)
+            .join(AlertRule, AlertEvent.rule_id == AlertRule.id)
+            .where(AlertEvent.transitioned_at >= since))
+    if rule_id:
+        stmt = stmt.where(AlertEvent.rule_id == rule_id)
+    stmt = keyset(stmt, AlertEvent.transitioned_at, AlertEvent.id, before, limit)
+    return [tuple(row) for row in (await db.execute(stmt)).all()]
+
+
+async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
+    """Every window, latest start first."""
+    return list((await db.execute(
+        select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
+    )).scalars())
+
+
+async def resources_for(db, source_module: str) -> list[str]:
+    """Sorted distinct resource names for a source module.
+
+    For monitor types (icmp/tcp/dns/http), supplements with monitor names so
+    the list is populated even before any metrics have been recorded.
+    """
+    result = await db.execute(
+        select(PluginMetric.resource_name)
+        .where(PluginMetric.source_module == source_module)
+        .distinct()
+        .order_by(PluginMetric.resource_name)
+    )
+    resources: set[str] = {r for (r,) in result}
+    if source_module in MONITOR_TYPE_VALUES:
+        mon_result = await db.execute(
+            select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name))
+        resources.update(name for (name,) in mon_result)
+    return sorted(resources)
+
+
+async def metrics_for(db, source_module: str) -> list[str]:
+    """The metric names a rule on this source can watch: the static catalog, or
+    for dynamic sources (snmp) the names recorded so far."""
+    metrics = metric_catalog().get(source_module, [])
+    if source_module and not metrics:
+        result = await db.execute(
+            select(PluginMetric.metric_name)
+            .where(PluginMetric.source_module == source_module)
+            .distinct()
+            .order_by(PluginMetric.metric_name))
+        metrics = [r for (r,) in result]
+    return metrics
+
+
+def _choices(values: list[str], limit: int = 20) -> str:
+    if not values:
+        return "none yet"
+    shown = ", ".join(values[:limit])
+    return shown + (f" (and {len(values) - limit} more)" if len(values) > limit else "")
+
+
+async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
+                      operator: str, threshold, consecutive_failures_required=1,
+                      consecutive_ok_required=1, notify_no_data=True) -> dict:
+    """Validated column values for a new or edited alert rule.
+
+    The same choices the rule form offers: a known source, a resource that
+    source has reported (or, for monitors, a monitor of that type), and a
+    metric it emits. ValueError says what was wrong and what is valid.
+    """
+    name = (name or "").strip()
+    if not name:
+        raise ValueError("an alert rule needs a name.")
+    source_module = (source_module or "").strip()
+    catalog = metric_catalog()
+    if source_module not in catalog:
+        raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
+    resource_name = (resource_name or "").strip()
+    resources = await resources_for(db, source_module)
+    if resource_name not in resources:
+        raise ValueError(f"{source_module} has no resource {resource_name!r}. "
+                         f"Known: {_choices(resources)}.")
+    metric_name = (metric_name or "").strip()
+    metrics = await metrics_for(db, source_module)
+    if metric_name not in metrics:
+        raise ValueError(f"{source_module} does not report {metric_name!r}. "
+                         f"Its metrics: {_choices(metrics)}.")
+    try:
+        op = AlertOperator((operator or "").strip())
+    except ValueError:
+        raise ValueError(f"operator {operator!r} is not one of "
+                         f"{', '.join(o.value for o in AlertOperator)}.") from None
+    try:
+        threshold = float(threshold)
+    except (TypeError, ValueError):
+        raise ValueError(f"threshold must be a number, got {threshold!r}.") from None
+    try:
+        consecutive = (1 if consecutive_failures_required in (None, "")
+                       else int(consecutive_failures_required))
+    except (TypeError, ValueError):
+        raise ValueError("consecutive checks must be a whole number.") from None
+    if consecutive < 1:
+        raise ValueError("consecutive checks must be at least 1.")
+    try:
+        ok_checks = (1 if consecutive_ok_required in (None, "")
+                     else int(consecutive_ok_required))
+    except (TypeError, ValueError):
+        raise ValueError("checks to clear must be a whole number.") from None
+    if ok_checks < 1:
+        raise ValueError("checks to clear must be at least 1.")
+    return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
+            "metric_name": metric_name, "operator": op, "threshold": threshold,
+            "consecutive_failures_required": consecutive,
+            "consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)}
+
+
+# Changing any of these changes what the rule is watching, so whatever state
+# it had built up no longer means anything.
+WATCHED_FIELDS = ("source_module", "resource_name", "metric_name", "operator", "threshold")
+
+
+async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool | None = None) -> None:
+    """Apply validated fields (and optionally enabled) to a rule, in the
+    caller's transaction. A change to what it watches, or disabling or
+    re-enabling it, puts its alert back to inactive and starts its silence
+    watch over (core.alerts.reset_rule_state)."""
+    from steward.core.alerts import reset_rule_state
+    changed = [k for k in WATCHED_FIELDS if k in fields and getattr(rule, k) != fields[k]]
+    for k, v in fields.items():
+        setattr(rule, k, v)
+    disabling = enabled is False and rule.enabled
+    enabling = enabled is True and not rule.enabled
+    if enabled is not None:
+        rule.enabled = bool(enabled)
+    if disabling:
+        await reset_rule_state(db, rule, "rule disabled")
+    elif enabling:
+        await reset_rule_state(db, rule, "rule enabled")
+    elif changed:
+        await reset_rule_state(db, rule, "rule changed: " + ", ".join(changed))
+
+
+async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
+                   ansible_action: dict | None = None) -> AlertRule:
+    """Insert a rule and its inactive state row, in the caller's transaction."""
+    rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
+    db.add(rule)
+    await db.flush()
+    db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
+                      last_transition_at=datetime.now(timezone.utc)))
+    return rule
+
+
+async def acknowledge(db, rule_id: str, user_id: str) -> AlertState:
+    """Acknowledge a firing alert, in the caller's transaction.
+
+    LookupError when the rule has no state; ValueError when it is not firing.
+    """
+    state = (await db.execute(
+        select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one_or_none()
+    if state is None:
+        raise LookupError(rule_id)
+    if state.state != AlertStateEnum.firing:
+        raise ValueError(f"the alert is {state.state.value}, not firing; only a firing alert "
+                         "can be acknowledged.")
+    now = datetime.now(timezone.utc)
+    state.state = AlertStateEnum.acknowledged
+    state.acknowledged_by = user_id
+    state.acknowledged_at = now
+    state.last_transition_at = now
+    return state
+
+
+def window_fields(*, name: str, start_at: datetime, end_at: datetime,
+                  scope_source: str | None, scope_resource: str | None) -> dict:
+    """Validated columns for a maintenance window. A resource needs a source."""
+    name = (name or "").strip()
+    if not name:
+        raise ValueError("a maintenance window needs a name.")
+    if end_at <= start_at:
+        raise ValueError("the window must end after it starts.")
+    scope_source = (scope_source or "").strip() or None
+    scope_resource = (scope_resource or "").strip() or None
+    catalog = metric_catalog()
+    if scope_source is not None and scope_source not in catalog:
+        raise ValueError(f"source {scope_source!r} is not one of {', '.join(catalog)}.")
+    if scope_source is None:
+        scope_resource = None  # can't have resource without source
+    return {"name": name[:128], "start_at": start_at, "end_at": end_at,
+            "scope_source": scope_source, "scope_resource": scope_resource}
+
+
+def window_scope(w) -> str:
+    if not w.scope_source:
+        return "all"
+    return f"{w.scope_source}/{w.scope_resource}" if w.scope_resource else f"{w.scope_source}/*"

steward/ansible/bundled/host_agent/install.yml

@@ -61,6 +61,8 @@
       notify: restart steward-agent
 
     - name: Write agent config
+      # The content carries steward_token; --diff and a failure would print it.
+      no_log: true
       ansible.builtin.copy:
         dest: /etc/steward-agent.conf
         owner: root

steward/ansible/bundled/host_agent/provision.yml

@@ -117,6 +117,8 @@
       notify: restart steward-agent
 
     - name: Write agent config
+      # The content carries steward_token; --diff and a failure would print it.
+      no_log: true
       ansible.builtin.copy:
         dest: /etc/steward-agent.conf
         owner: root

steward/ansible/bundled/maintenance/docker_prune.yml

@@ -1,29 +1,64 @@
 ---
-# description: Reclaim disk on Docker / Swarm nodes by pruning unused images, containers, networks and build cache.
+# description: Reclaim disk on Docker / Swarm nodes — prune stopped containers, unused images, or a full system prune.
 # steward:category: maintenance
 # steward:confirm: true
 # Reclaim disk on Docker / Docker Swarm nodes by removing unused data.
-# Safe by default: prunes dangling images, stopped containers, unused networks
-# and build cache. Set extra-vars to widen scope:
-#   prune_all_images=true   also remove ALL unused images (not just dangling)
-#   prune_volumes=true      also remove unused named volumes (data loss risk)
-- name: Docker system prune
+# `prune_target` selects the scope (default `system` preserves the original
+# behavior for existing manual/scheduled callers that don't set it):
+#   prune_target=containers   remove stopped containers only (docker container prune)
+#   prune_target=images       remove unused images (docker image prune;
+#                             + prune_all_images=true → -a, i.e. ALL unused, not just dangling)
+#   prune_target=system       docker system prune (dangling images, stopped
+#                             containers, unused networks, build cache). Widen with:
+#                               prune_all_images=true   also ALL unused images
+#                               prune_volumes=true       also unused named volumes (data loss risk)
+- name: Docker prune
   hosts: all
   gather_facts: false
   become: true
   vars:
+    prune_target: system
     prune_all_images: false
     prune_volumes: false
   tasks:
+    - name: Validate prune_target
+      ansible.builtin.assert:
+        that: prune_target in ['containers', 'images', 'system']
+        fail_msg: "prune_target must be one of: containers, images, system (got '{{ prune_target }}')"
+        quiet: true
+
+    # ── Stopped containers only ───────────────────────────────────────────────
+    - name: Prune stopped containers
+      ansible.builtin.command:
+        argv: ['docker', 'container', 'prune', '-f']
+      register: container_prune
+      changed_when: "'Total reclaimed space: 0B' not in container_prune.stdout"
+      when: prune_target == 'containers'
+
+    # ── Unused images (dangling, or all unused with -a) ───────────────────────
+    - name: Prune unused images
+      ansible.builtin.command:
+        argv: >-
+          {{ ['docker', 'image', 'prune', '-f']
+             + (['-a'] if prune_all_images | bool else []) }}
+      register: image_prune
+      changed_when: "'Total reclaimed space: 0B' not in image_prune.stdout"
+      when: prune_target == 'images'
+
+    # ── Full system prune (default) ───────────────────────────────────────────
     - name: Run docker system prune
       ansible.builtin.command:
         argv: >-
           {{ ['docker', 'system', 'prune', '-f']
              + (['-a'] if prune_all_images | bool else [])
              + (['--volumes'] if prune_volumes | bool else []) }}
-      register: prune_result
-      changed_when: "'Total reclaimed space: 0B' not in prune_result.stdout"
+      register: system_prune
+      changed_when: "'Total reclaimed space: 0B' not in system_prune.stdout"
+      when: prune_target == 'system'
 
     - name: Report reclaimed space
       ansible.builtin.debug:
-        msg: "{{ prune_result.stdout_lines | select | list }}"
+        msg: >-
+          {{ ((container_prune.stdout_lines | default([]))
+              + (image_prune.stdout_lines | default([]))
+              + (system_prune.stdout_lines | default([]))) | select | list }}

steward/ansible/executor.py

@@ -6,6 +6,7 @@ import logging
 import os
 import re
 import shutil
+import signal
 import tempfile
 import time
 from dataclasses import dataclass
@@ -26,6 +27,8 @@ _FLUSH_SECS = 5.0
 _MEM_LINES_CAP = 5000   # bounded per-run in-memory replay buffer (SSE late-joiners)
 _MEM_TTL_SECS = 300     # drop in-memory state this long after a run completes
 _FAILURE_CAP = 50       # max failed-task lines captured into structured results
+_READ_CHUNK = 65536     # output is read in chunks, never a whole line at once (#5156)
+_MAX_LINE_BYTES = 65536  # a longer line is cut here and marked, not allowed to stall
 
 # Full run logs persist here (survive restart + the 1 MB DB cap). Override via env.
 ARTIFACT_DIR = os.environ.get("STEWARD_ANSIBLE_LOG_DIR", "/data/ansible/runs")
@@ -35,6 +38,112 @@ def artifact_path(run_id: str) -> str:
     return os.path.join(ARTIFACT_DIR, f"{run_id}.log")
 
 
+# ── One run per host (#5158) ──────────────────────────────────────────────────
+# Two runs against the same host would race each other's changes, so a run
+# claims every host it touches before it takes a run slot, and waits (queued)
+# while another run holds one. The claim is all-or-nothing, so two runs over
+# overlapping hosts can't each hold half and wait on the other.
+
+HOST_WAIT_SECONDS = 3600.0  # a run waiting longer than this for a host fails
+
+
+class HostBusy(Exception):
+    """A run gave up waiting for a host another run held."""
+
+
+def run_hosts(inventory_content: str | None, inventory_path: str | None) -> set[str]:
+    """What a run touches, as claim keys: the address of each host in the YAML
+    inventory Steward wrote for it, or, for a source's own inventory file
+    (whose hosts Steward doesn't read), that file."""
+    if not inventory_content:
+        return {f"inventory:{inventory_path}"} if inventory_path else set()
+    import yaml
+    try:
+        data = yaml.safe_load(inventory_content)
+    except yaml.YAMLError:
+        return set()  # ansible-playbook will refuse it and say why
+    # A host listed again under a group carries no vars there, so its address
+    # is whichever listing gives one.
+    addresses: dict[str, str | None] = {}
+
+    def walk(group) -> None:
+        if not isinstance(group, dict):
+            return
+        members = group.get("hosts")
+        if isinstance(members, dict):
+            for name, hostvars in members.items():
+                addr = hostvars.get("ansible_host") if isinstance(hostvars, dict) else None
+                addresses[str(name)] = addr or addresses.get(str(name))
+        children = group.get("children")
+        if isinstance(children, dict):
+            for child in children.values():
+                walk(child)
+
+    if isinstance(data, dict):
+        for group in data.values():
+            walk(group)
+    return {str(addr or name).strip().lower() for name, addr in addresses.items()}
+
+
+def _host_gate(app: "Quart") -> dict:
+    """The claims and the wake-up event, bound to the running loop like the
+    run-slot semaphore (one loop in prod, one per test)."""
+    loop = asyncio.get_running_loop()
+    gate = getattr(app, "_ansible_host_gate", None)
+    if gate is None or gate["loop"] is not loop:
+        gate = {"loop": loop, "held": {}, "changed": asyncio.Event()}
+        app._ansible_host_gate = gate
+    return gate
+
+
+async def _claim_hosts(app: "Quart", run_id: str, hosts: set[str]) -> str | None:
+    """Hold every host in hosts for run_id, waiting while another run holds
+    any of them. Returns the line saying what it waited for, or None when it
+    didn't wait. Raises _Aborted when the run is cancelled while waiting,
+    HostBusy after HOST_WAIT_SECONDS."""
+    from steward.models.ansible import AnsibleRunStatus
+
+    gate = _host_gate(app)
+    held: dict[str, str] = gate["held"]
+    give_up = time.monotonic() + HOST_WAIT_SECONDS
+    waited: str | None = None
+    while True:
+        busy = sorted({held[h] for h in hosts if h in held and held[h] != run_id})
+        if not busy:
+            for h in hosts:
+                held[h] = run_id
+            return waited
+        if run_id in _cancelled:
+            raise _Aborted
+        remaining = give_up - time.monotonic()
+        if remaining <= 0:
+            names = ", ".join(sorted(h for h in hosts if h in held))
+            raise HostBusy(f"gave up after {HOST_WAIT_SECONDS:.0f} s waiting for "
+                           f"{names}, held by run {', '.join(busy)}")
+        if waited is None:
+            waited = (f"[queued — waiting for run {', '.join(busy)} "
+                      "to finish on the same host]")
+            await _set_status(app, run_id, AnsibleRunStatus.queued)
+            _broadcast(run_id, waited)
+        changed = gate["changed"]
+        try:
+            # Woken by a release; the short timeout also notices a cancel.
+            await asyncio.wait_for(changed.wait(), timeout=min(remaining, 2.0))
+        except asyncio.TimeoutError:
+            pass
+
+
+def _release_hosts(app: "Quart", run_id: str, hosts: set[str]) -> None:
+    gate = getattr(app, "_ansible_host_gate", None)
+    if gate is None:
+        return
+    for h in hosts:
+        if gate["held"].get(h) == run_id:
+            del gate["held"][h]
+    changed, gate["changed"] = gate["changed"], asyncio.Event()
+    changed.set()
+
+
 @dataclass
 class _Done:
     status: str
@@ -261,20 +370,63 @@ async def _set_status(app: "Quart", run_id: str, status) -> None:
             )
 
 
-async def _terminate_then_kill(proc: asyncio.subprocess.Process, grace: float = 10.0) -> None:
+def _signal_group(proc: asyncio.subprocess.Process, sig: int) -> None:
+    """Send sig to the run's whole process group (ansible-playbook and the
+    ssh and worker processes it forks), or to the process alone when it has
+    no group of its own."""
     try:
-        proc.terminate()
+        os.killpg(proc.pid, sig)
     except ProcessLookupError:
-        return
-    try:
-        await asyncio.wait_for(proc.wait(), timeout=grace)
-    except asyncio.TimeoutError:
+        pass
+    except OSError:
         try:
-            proc.kill()
+            proc.send_signal(sig)
         except ProcessLookupError:
             pass
 
 
+async def _terminate_then_kill(proc: asyncio.subprocess.Process, grace: float = 10.0) -> None:
+    """SIGTERM the run's process group, then SIGKILL it if it outlives grace.
+    The group is killed even once the leader has exited, so no worker it
+    forked is left behind (#5156)."""
+    if proc.returncode is None:
+        _signal_group(proc, signal.SIGTERM)
+        try:
+            await asyncio.wait_for(proc.wait(), timeout=grace)
+        except asyncio.TimeoutError:
+            pass
+    _signal_group(proc, signal.SIGKILL)
+
+
+async def _lines(stream: asyncio.StreamReader):
+    """The stream's lines, read in chunks so no line length can stall or break
+    the reader: a line past _MAX_LINE_BYTES is cut there and marked, and the
+    rest of it is dropped up to its newline (#5156)."""
+    pending = b""
+    skipping = False
+    while True:
+        chunk = await stream.read(_READ_CHUNK)
+        if not chunk:
+            break
+        pending += chunk
+        while True:
+            nl = pending.find(b"\n")
+            if nl < 0:
+                break
+            line, pending = pending[:nl], pending[nl + 1:]
+            if skipping:
+                skipping = False
+                continue
+            yield line
+        if len(pending) > _MAX_LINE_BYTES and not skipping:
+            yield pending[:_MAX_LINE_BYTES] + b" [line cut at %d bytes]" % _MAX_LINE_BYTES
+            pending, skipping = b"", True
+        elif skipping:
+            pending = b""
+    if pending and not skipping:
+        yield pending
+
+
 def cancel_run(run_id: str) -> bool:
     """Request cancellation of a run. Terminates the process if it's running;
     a queued run is flagged so start_run aborts before launch. Returns False
@@ -293,6 +445,45 @@ def cancel_run(run_id: str) -> bool:
 
 # ── Run execution ─────────────────────────────────────────────────────────────
 
+# ── Secret scrubbing ──────────────────────────────────────────────────────────
+# Ansible echoes values back: a failed task prints its args, -v prints vars,
+# --diff prints templated files. Each line is masked before it is broadcast,
+# written to the log artifact or stored on the run, so a secret the run was
+# handed never leaves it.
+MASK = "***"
+_MIN_SECRET_LEN = 4  # masking "on" or "1" would mangle the output and hide nothing
+
+
+def _strings(value) -> list[str]:
+    if isinstance(value, str):
+        return [value]
+    if isinstance(value, dict):
+        return [s for v in value.values() for s in _strings(v)]
+    if isinstance(value, (list, tuple, set)):
+        return [s for v in value for s in _strings(v)]
+    return []
+
+
+def run_secrets(creds: dict | None, connection: dict | None,
+                secret_vars: dict | None, redact: list[str] | None) -> list[str]:
+    """Every secret value this run is handed: the global become/vault
+    passwords, the bootstrap passwords, the sensitive run variables and any
+    the caller names (a per-host agent token)."""
+    creds, connection = creds or {}, connection or {}
+    values = [creds.get("become_password"), creds.get("vault_password"),
+              connection.get("password"), connection.get("become_password"),
+              *_strings(secret_vars), *(redact or [])]
+    found = {v for v in values if isinstance(v, str) and len(v) >= _MIN_SECRET_LEN}
+    return sorted(found, key=len, reverse=True)  # longest first: no partial masks
+
+
+def scrub(line: str, secrets: list[str]) -> str:
+    for value in secrets:
+        if value in line:
+            line = line.replace(value, MASK)
+    return line
+
+
 async def start_run(
     app: "Quart",
     run_id: str,
@@ -303,11 +494,13 @@ async def start_run(
     inventory_content: str | None = None,
     connection: dict | None = None,
     secret_vars: dict | None = None,
+    redact: list[str] | None = None,
 ) -> None:
     """Execute ansible-playbook as a subprocess and update the DB run row.
 
-    Respects a global concurrency cap (app._ansible_semaphore): if no slot is
-    free the run shows as 'queued' until one frees. Supports cancellation, a
+    Waits (as 'queued') while another run holds one of its hosts (#5158), then
+    for a slot under the global concurrency cap (app._ansible_semaphore).
+    Supports cancellation, a
     persistent full-log artifact, and a parsed per-host result summary.
 
     connection is an optional per-run SSH override (user/password) for
@@ -315,10 +508,12 @@ async def start_run(
     never persisted on the AnsibleRun row, never placed on argv, and not logged.
     secret_vars are run-time playbook variables flagged sensitive; like
     connection they reach the subprocess (merged into the extra-vars file) but
-    are never persisted.
+    are never persisted. Every value in run_secrets() — redact names more — is
+    masked in each output line before it goes anywhere.
     """
     from steward.models.ansible import AnsibleRunStatus
 
+    secrets = run_secrets(app.config.get("ANSIBLE", {}), connection, secret_vars, redact)
     _run_lines[run_id] = []
     db_output = ""
     truncated = False
@@ -354,15 +549,30 @@ async def start_run(
         sem = asyncio.Semaphore(max(1, cap))
         app._ansible_semaphore = sem
         app._ansible_sem_loop = loop
-    queued = sem.locked()
-    if queued:
-        await _set_status(app, run_id, AnsibleRunStatus.queued)
-        _broadcast(run_id, "[queued — waiting for an Ansible run slot]")
-    await sem.acquire()
+    hosts = run_hosts(inventory_content, inventory_path)
 
     final_status = "failed"
     logf = None
+    proc = None
+    interrupted = False
+    slot_held = False
     try:
+        # The hosts first, so a run waiting on a host doesn't sit on a run slot.
+        # What it waited for goes in the stored output too, so a run that sat
+        # queued says why after the live view is gone.
+        waits: list[str] = []
+        host_wait = await _claim_hosts(app, run_id, hosts)
+        if host_wait:
+            waits.append(host_wait)
+        if sem.locked():
+            if not waits:
+                await _set_status(app, run_id, AnsibleRunStatus.queued)
+            waits.append("[queued — waiting for an Ansible run slot]")
+            _broadcast(run_id, waits[-1])
+        queued = bool(waits)
+        db_output = "\n".join(waits)
+        await sem.acquire()
+        slot_held = True
         if run_id in _cancelled:  # cancelled while queued — abort before launching
             raise _Aborted
         if queued:
@@ -421,18 +631,20 @@ async def start_run(
                 if ssh_user:
                     user_args += ["--user", ssh_user]
 
+            # Its own session, so the whole group (ssh, forks) can be signalled.
             proc = await asyncio.create_subprocess_exec(
                 *cmd, *cred_args, *boot_args, *user_args, *ev_args,
                 stdout=asyncio.subprocess.PIPE,
                 stderr=asyncio.subprocess.STDOUT,
                 cwd=cwd,
                 env=ansible_env(creds, os.environ),
+                start_new_session=True,
             )
             _run_procs[run_id] = proc
             assert proc.stdout is not None
 
-            async for raw_line in proc.stdout:
-                line = raw_line.decode(errors="replace").rstrip("\n\r")
+            async for raw_line in _lines(proc.stdout):
+                line = scrub(raw_line.decode(errors="replace").rstrip("\n\r"), secrets)
                 _broadcast(run_id, line)
                 if logf is not None:
                     logf.write(line + "\n")
@@ -454,7 +666,8 @@ async def start_run(
 
             await proc.wait()
             recap = parse_recap(_run_lines.get(run_id, []))
-            if run_id in _cancelled:
+            # A cancel that lands after the run finished doesn't undo its result.
+            if run_id in _cancelled and proc.returncode != 0:
                 final_status = "cancelled"
             elif proc.returncode != 0:
                 final_status = "failed"
@@ -469,16 +682,27 @@ async def start_run(
             else:
                 final_status = "success"
         finally:
+            # Whatever ended the read (the run finishing, an error, the task
+            # being cancelled), nothing the run started outlives it.
+            if proc is not None:
+                await asyncio.shield(_terminate_then_kill(proc, grace=5.0))
             shutil.rmtree(tmpdir, ignore_errors=True)
 
     except _Aborted:
         final_status = "cancelled"
+    except asyncio.CancelledError:
+        # The task itself was cancelled (shutdown): record that the run did
+        # not finish rather than leave it marked running (#5156).
+        final_status, interrupted = "cancelled", True
+        msg = "[run error] Steward stopped the run before it finished."
+        _broadcast(run_id, msg)
+        db_output = (db_output + "\n" + msg) if db_output else msg
     except Exception as exc:
         logger.exception("Ansible run %s failed with exception", run_id)
         final_status = "cancelled" if run_id in _cancelled else "failed"
         # Surface WHY in the run output instead of an empty "failed" (e.g. ENOSPC
         # when the temp dir can't be created — the run never reaches Ansible).
-        msg = f"[run error] {type(exc).__name__}: {exc}"
+        msg = scrub(f"[run error] {type(exc).__name__}: {exc}", secrets)
         _broadcast(run_id, msg)
         if len(failures) < _FAILURE_CAP:
             failures.append(msg[:500])
@@ -488,8 +712,16 @@ async def start_run(
             logf.close()
         _run_procs.pop(run_id, None)
         _cancelled.discard(run_id)
-        sem.release()
+        if slot_held:
+            sem.release()
+        _release_hosts(app, run_id, hosts)
 
     results = {"hosts": parse_recap(_run_lines.get(run_id, [])), "failures": failures}
-    await _flush(final_status, results=results)
+    try:
+        await asyncio.shield(_flush(final_status, results=results))
+    except Exception:
+        logger.exception("Ansible run %s: could not record its final status %s",
+                         run_id, final_status)
     _broadcast(run_id, _Done(status=final_status))
+    if interrupted:
+        raise asyncio.CancelledError

steward/ansible/guards.py

@@ -0,0 +1,86 @@
+"""What a playbook run may be pointed at, and which variables only an admin
+may set.
+
+Ansible runs inside the Steward container, so a run is code on the Steward
+server unless it is held to playbooks its source lists and to variables that
+only shape what happens on the target. An operator may run playbooks and set
+their variables. Three things would let one run commands on Steward itself,
+and are refused for anyone but an admin:
+
+- a playbook or inventory path outside what the source lists;
+- an `ansible_*` variable that changes how or where Ansible connects
+  (`ansible_connection: local`, `ansible_ssh_common_args` with a
+  ProxyCommand, an interpreter or executable path);
+- a Jinja expression in any value (`{{ lookup('pipe', …) }}` is evaluated
+  on the controller when the value is used).
+
+Playbook files themselves are admin-only to write (ansible/routes.py).
+"""
+from __future__ import annotations
+
+from steward.ansible import sources as src_module
+from steward.models.users import UserRole
+
+# The ansible_* variables an operator may set: which address, port and user
+# to reach a target as, and whether to become another user there.
+OPERATOR_ANSIBLE_VARS = frozenset({
+    "ansible_host", "ansible_port", "ansible_user", "ansible_password",
+    "ansible_become", "ansible_become_user", "ansible_become_password",
+})
+_JINJA = ("{{", "{%", "{#")
+
+
+def check_playbook(source: dict, playbook_path: str) -> str | None:
+    """None when the source lists this playbook, else why it is refused."""
+    if playbook_path not in src_module.discover_playbooks(source["path"]):
+        return f"Source {source['name']!r} has no playbook {playbook_path!r}"
+    return None
+
+
+def check_inventory_scope(source: dict, scope: str) -> str | None:
+    """None for a Steward scope or an inventory file the source has, else why not."""
+    if scope.startswith("steward:"):
+        return None
+    if scope.startswith("repo:"):
+        parts = scope.split(":", 2)
+        path = parts[2] if len(parts) == 3 else ""
+        if path in src_module.discover_inventories(source["path"]):
+            return None
+        return f"Source {source['name']!r} has no inventory file {path!r}"
+    return "Invalid inventory_scope"
+
+
+def _has_jinja(value) -> bool:
+    if isinstance(value, str):
+        return any(mark in value for mark in _JINJA)
+    if isinstance(value, dict):
+        return any(_has_jinja(k) or _has_jinja(v) for k, v in value.items())
+    if isinstance(value, (list, tuple)):
+        return any(_has_jinja(v) for v in value)
+    return False
+
+
+def check_vars_for_role(variables: dict | None, role: UserRole | None) -> str | None:
+    """None when `role` may set these variables, else a message naming what
+    only an admin may set. role=None is treated as the least privileged."""
+    if role == UserRole.admin or not variables:
+        return None
+    names = sorted(n for n in variables
+                   if str(n).startswith("ansible_") and n not in OPERATOR_ANSIBLE_VARS)
+    if names:
+        return (f"Only an admin can set {', '.join(names)}: these change how or where "
+                "Ansible connects, and can run commands on the Steward server itself.")
+    templated = sorted(str(k) for k, v in variables.items() if _has_jinja(k) or _has_jinja(v))
+    if templated:
+        return (f"Only an admin can set a value containing {{{{ }}}}, {{% %}} or {{# #}} "
+                f"({', '.join(templated)}): Ansible evaluates it on the Steward server.")
+    return None
+
+
+
+def changed_vars(new: dict | None, old: dict | None) -> dict:
+    """The entries of `new` that are added or differ from `old`. An edit is
+    held to the editor's role only for what it changes, so an operator can
+    still edit a group whose connection vars an admin set."""
+    old = old or {}
+    return {k: v for k, v in (new or {}).items() if k not in old or old[k] != v}

steward/ansible/inventory_gen.py

@@ -106,3 +106,48 @@ async def fetch_scope_targets(db, scope: str) -> list:
 
     result = await db.execute(stmt)
     return list(result.scalars().all())
+
+
+def describe_scope(scope: str, targets: dict[str, str], groups: dict[str, str]) -> str:
+    """A run's inventory scope in words: "target web-1", "group Docker Swarm",
+    "all inventory targets", "<source>/<inventory file>". `targets` and
+    `groups` map id → name; a scope naming a deleted one says so."""
+    scope = scope or ""
+    if scope == "steward:all":
+        return "all inventory targets"
+    for prefix, kind, names in (("steward:target:", "target", targets),
+                                ("steward:group:", "group", groups)):
+        if scope.startswith(prefix):
+            ident = scope[len(prefix):]
+            name = names.get(ident)
+            return f"{kind} {name}" if name else f"{kind} {ident[:8]} (deleted)"
+    if scope.startswith("repo:"):
+        source, _, inv = scope[len("repo:"):].rpartition(":")
+        return f"{source}/{inv}" if source else inv
+    return scope
+
+
+def run_title(playbook: str, scope_label: str | None) -> str:
+    """A run named by what it ran where: "update.yml on web-1",
+    "site.yml on group Docker Swarm", "ping.yml on all inventory targets"."""
+    where = scope_label or ""
+    if where.startswith("target "):
+        where = where[len("target "):]
+    return f"{playbook} on {where}" if where else playbook
+
+
+async def scope_labels(db, scopes) -> dict[str, str]:
+    """{scope: describe_scope(scope)} for the given scopes, in two queries."""
+    from sqlalchemy import select
+    from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget
+
+    scopes = {s for s in scopes if s}
+    t_ids = {s[len("steward:target:"):] for s in scopes if s.startswith("steward:target:")}
+    g_ids = {s[len("steward:group:"):] for s in scopes if s.startswith("steward:group:")}
+    targets = dict((await db.execute(
+        select(AnsibleTarget.id, AnsibleTarget.name).where(AnsibleTarget.id.in_(t_ids))
+    )).all()) if t_ids else {}
+    groups = dict((await db.execute(
+        select(AnsibleGroup.id, AnsibleGroup.name).where(AnsibleGroup.id.in_(g_ids))
+    )).all()) if g_ids else {}
+    return {s: describe_scope(s, targets, groups) for s in scopes}

steward/ansible/inventory_routes.py

@@ -3,11 +3,13 @@ from __future__ import annotations
 import uuid
 import yaml
 
-from quart import Blueprint, current_app, redirect, render_template, request, url_for
+from quart import Blueprint, current_app, redirect, render_template, request, session, url_for
 from sqlalchemy import select
 from sqlalchemy.orm import selectinload
 
+from steward.ansible import guards
 from steward.auth.middleware import require_role
+from steward.core.audit import audit_request as _audit
 from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget
 from steward.models.users import UserRole
 
@@ -30,6 +32,12 @@ def _parse_ansible_vars(raw: str) -> dict:
     return parsed
 
 
+def _vars_refusal(new: dict, old: dict | None = None) -> str | None:
+    """Why this editor may not make this vars change, or None (ansible/guards.py)."""
+    return guards.check_vars_for_role(guards.changed_vars(new, old),
+                                      UserRole(session.get("user_role", "viewer")))
+
+
 # ---------------------------------------------------------------------------
 # Groups
 # ---------------------------------------------------------------------------
@@ -59,10 +67,16 @@ async def groups_create():
     except ValueError as exc:
         return str(exc), 400
 
+    err = _vars_refusal(ansible_vars)
+    if err:
+        return err, 400
+
     group = AnsibleGroup(id=str(uuid.uuid4()), name=name, ansible_vars=ansible_vars)
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
             db.add(group)
+    await _audit("ansible.group.created", "ansible_group", group.id,
+                 {"name": name, "vars": sorted(ansible_vars)})
     return redirect(url_for("ansible_inventory.groups_list"))
 
 
@@ -111,6 +125,9 @@ async def group_update(group_id: str):
             group = result.scalar_one_or_none()
             if group is None:
                 return "Group not found", 404
+            err = _vars_refusal(ansible_vars, group.ansible_vars)
+            if err:
+                return err, 400
 
             name = form.get("name", group.name).strip()
             if name:
@@ -123,7 +140,10 @@ async def group_update(group_id: str):
             ).scalars().all()
             target_map = {t.id: t for t in all_targets}
             group.targets = [target_map[tid] for tid in new_ids if tid in target_map]
+            detail = {"name": group.name, "vars": sorted(ansible_vars),
+                      "targets": sorted(t.name for t in group.targets)}
 
+    await _audit("ansible.group.updated", "ansible_group", group_id, detail)
     return redirect(url_for("ansible_inventory.group_detail", group_id=group_id))
 
 
@@ -136,8 +156,11 @@ async def group_delete(group_id: str):
                 select(AnsibleGroup).where(AnsibleGroup.id == group_id)
             )
             group = result.scalar_one_or_none()
-            if group is not None:
-                await db.delete(group)
+            if group is None:
+                return redirect(url_for("ansible_inventory.groups_list"))
+            name = group.name
+            await db.delete(group)
+    await _audit("ansible.group.deleted", "ansible_group", group_id, {"name": name})
     return redirect(url_for("ansible_inventory.groups_list"))
 
 
@@ -171,12 +194,18 @@ async def targets_create():
     except ValueError as exc:
         return str(exc), 400
 
+    err = _vars_refusal(ansible_vars)
+    if err:
+        return err, 400
+
     target = AnsibleTarget(
         id=str(uuid.uuid4()), name=name, address=address, ansible_vars=ansible_vars
     )
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
             db.add(target)
+    await _audit("ansible.target.created", "ansible_target", target.id,
+                 {"name": name, "address": address, "vars": sorted(ansible_vars)})
     return redirect(url_for("ansible_inventory.targets_list"))
 
 
@@ -225,6 +254,9 @@ async def target_update(target_id: str):
             target = result.scalar_one_or_none()
             if target is None:
                 return "Target not found", 404
+            err = _vars_refusal(ansible_vars, target.ansible_vars)
+            if err:
+                return err, 400
 
             name = form.get("name", target.name).strip()
             address = form.get("address", target.address).strip()
@@ -238,7 +270,10 @@ async def target_update(target_id: str):
             all_groups_result = await db.execute(select(AnsibleGroup))
             group_map = {g.id: g for g in all_groups_result.scalars().all()}
             target.groups = [group_map[gid] for gid in new_group_ids if gid in group_map]
+            detail = {"name": target.name, "address": target.address, "vars": sorted(ansible_vars),
+                      "groups": sorted(g.name for g in target.groups)}
 
+    await _audit("ansible.target.updated", "ansible_target", target_id, detail)
     return redirect(url_for("ansible_inventory.target_detail", target_id=target_id))
 
 
@@ -251,6 +286,9 @@ async def target_delete(target_id: str):
                 select(AnsibleTarget).where(AnsibleTarget.id == target_id)
             )
             target = result.scalar_one_or_none()
-            if target is not None:
-                await db.delete(target)
+            if target is None:
+                return redirect(url_for("ansible_inventory.targets_list"))
+            name = target.name
+            await db.delete(target)
+    await _audit("ansible.target.deleted", "ansible_target", target_id, {"name": name})
     return redirect(url_for("ansible_inventory.targets_list"))

steward/ansible/origins.py

@@ -0,0 +1,52 @@
+"""Who or what started each run, as the runs list and run detail say it.
+
+A run records the user who started it (triggered_by) and where it came from
+(origin / origin_id, migration 0031): a schedule, an alert rule, or an agent
+over MCP. This turns those into "schedule nightly", "alert rule nas down",
+"alice via MCP" or "alice", in three queries for a whole page of runs.
+"""
+from __future__ import annotations
+
+from sqlalchemy import select
+
+from steward.models.alerts import AlertRule
+from steward.models.ansible_schedule import AnsibleSchedule
+from steward.models.users import User
+
+
+def describe_origin(run, users: dict, schedules: dict, rules: dict) -> dict:
+    """{"text", "url", "kind"} for one run, from id → name maps. A schedule or
+    rule deleted since is named as such; a run with neither a user nor an
+    origin predates origins and was started by Steward itself."""
+    who = users.get(run.triggered_by, "a deleted user") if run.triggered_by else None
+    if run.origin == "schedule":
+        name = schedules.get(run.origin_id)
+        text = f"schedule {name}" if name else "a deleted schedule"
+        if who:
+            text += f" (run now by {who})"
+        return {"text": text, "url": "/ansible/schedules" if name else None, "kind": "schedule"}
+    if run.origin == "alert":
+        name = rules.get(run.origin_id)
+        return {"text": f"alert rule {name}" if name else "a deleted alert rule",
+                "url": f"/alerts/history?rule={run.origin_id}" if name else None, "kind": "alert"}
+    if run.origin == "mcp":
+        return {"text": f"{who or 'an agent'} via MCP", "url": None, "kind": "mcp"}
+    if who:
+        return {"text": who, "url": None, "kind": "user"}
+    return {"text": "Steward", "url": None, "kind": "system"}
+
+
+async def run_origins(db, runs) -> dict[str, dict]:
+    """{run id: describe_origin(run)} for a page of runs."""
+    runs = list(runs)
+    user_ids = {r.triggered_by for r in runs if r.triggered_by}
+    sched_ids = {r.origin_id for r in runs if r.origin == "schedule" and r.origin_id}
+    rule_ids = {r.origin_id for r in runs if r.origin == "alert" and r.origin_id}
+    users = dict((await db.execute(
+        select(User.id, User.username).where(User.id.in_(user_ids)))).all()) if user_ids else {}
+    schedules = dict((await db.execute(
+        select(AnsibleSchedule.id, AnsibleSchedule.name).where(AnsibleSchedule.id.in_(sched_ids)))).all()) \
+        if sched_ids else {}
+    rules = dict((await db.execute(
+        select(AlertRule.id, AlertRule.name).where(AlertRule.id.in_(rule_ids)))).all()) if rule_ids else {}
+    return {r.id: describe_origin(r, users, schedules, rules) for r in runs}

steward/ansible/routes.py

@@ -8,11 +8,15 @@ from quart import (
 )
 from sqlalchemy import select
 
-from steward.ansible import executor, sources as src_module
+from steward.ansible import executor, guards, sources as src_module
+from steward.ansible.inventory_gen import run_title, scope_labels
+from steward.ansible.origins import run_origins
 from steward.auth.middleware import require_role
+from steward.core.audit import audit_request as _audit
+from steward.core.paging import keyset, next_page
 from steward.models.ansible import AnsibleRun, AnsibleRunStatus
 from steward.models.ansible_schedule import AnsibleSchedule
-from steward.models.users import User, UserRole
+from steward.models.users import UserRole
 
 INTERVAL_PRESETS = [
     (300, "Every 5 minutes"),
@@ -34,12 +38,36 @@ def _get_sources() -> list[dict]:
 @ansible_bp.get("/")
 @require_role(UserRole.viewer)
 async def index():
+    """Every run, newest first, a page at a time as the reader scrolls;
+    ?before=<cursor> is the next page's rows alone (core.paging)."""
+    before = request.args.get("before")
     async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(AnsibleRun).order_by(AnsibleRun.started_at.desc()).limit(50)
-        )
-        runs = result.scalars().all()
-    return await render_template("ansible/index.html", runs=runs)
+        stmt = keyset(select(AnsibleRun), AnsibleRun.started_at, AnsibleRun.id, before)
+        runs, after = next_page((await db.execute(stmt)).scalars().all(),
+                                lambda r: (r.started_at, r.id))
+        labels = await scope_labels(db, [r.inventory_scope for r in runs])
+        origins = await run_origins(db, runs)
+    now = datetime.now(timezone.utc)
+    ctx = {"runs": [{**run_view(r, labels, now), "origin": origins[r.id]} for r in runs],
+           "after": after}
+    if before:
+        return await render_template("ansible/_runs_page.html", **ctx)
+    return await render_template("ansible/index.html", **ctx)
+
+
+def run_duration(run: AnsibleRun, now: datetime) -> float | None:
+    """Seconds a run took, or has taken so far while it is still going."""
+    if run.started_at is None:
+        return None
+    return ((run.finished_at or now) - run.started_at).total_seconds()
+
+
+def run_view(run: AnsibleRun, labels: dict[str, str], now: datetime) -> dict:
+    """A run as the pages name it: its title, scope label and duration."""
+    label = labels.get(run.inventory_scope, run.inventory_scope)
+    return {"run": run, "title": run_title(run.playbook_path, label), "scope_label": label,
+            "duration_s": run_duration(run, now),
+            "check": bool((run.params or {}).get("check"))}
 
 
 @ansible_bp.get("/browse")
@@ -104,6 +132,30 @@ async def view_playbook(source_name: str, playbook_path: str):
     )
 
 
+def _caller_role() -> UserRole:
+    return UserRole(session.get("user_role", "viewer"))
+
+
+def _run_vars(params: dict | None, secret_vars: dict | None = None) -> dict:
+    return {**((params or {}).get("extra_vars_map") or {}), **(secret_vars or {})}
+
+
+def check_run_form(form, source: dict | None, playbook_path: str,
+                   params: dict | None, secret_vars: dict | None) -> str | None:
+    """The server-side gates a web run must pass (the form's own checks are
+    only a convenience): who may set these variables, and the playbook's
+    steward:confirm flag, which a real (non-check) run must have ticked."""
+    err = guards.check_vars_for_role(_run_vars(params, secret_vars), _caller_role())
+    if err:
+        return err
+    content = src_module.read_playbook(source["path"], playbook_path) if source else None
+    meta = src_module.discover_playbook_meta(content or "")
+    if meta["confirm"] and not (params or {}).get("check") and "confirmed" not in form:
+        return (f"{playbook_path} is flagged destructive by its author (steward:confirm). "
+                "Tick the confirmation to run it, or run it in check mode.")
+    return None
+
+
 def _parse_run_params(form) -> tuple[dict | None, dict | None, str | None]:
     """Build executor params + the unpersisted secret-vars dict from form fields.
 
@@ -234,6 +286,10 @@ async def create_run():
         return "playbook_path is required", 400
 
     params_or_none, secret_vars, err = _parse_run_params(form)
+    if err:
+        return err, 400
+    source = next((s for s in _get_sources() if s["name"] == source_name), None)
+    err = check_run_form(form, source, playbook_path, params_or_none, secret_vars)
     if err:
         return err, 400
 
@@ -250,11 +306,17 @@ async def create_run():
         return err, 404
     if err:
         return err, 400
+    await _audit("ansible.run.started", "ansible_run", run.id, {
+        "source": source_name, "playbook": playbook_path, "scope": inventory_scope,
+        "check": bool((params_or_none or {}).get("check")),
+        "vars": sorted(_run_vars(params_or_none, secret_vars))})
 
+    async with current_app.db_sessionmaker() as db:
+        labels = await scope_labels(db, [run.inventory_scope])
     return await render_template(
         "ansible/run_started.html",
         run=run,
-        source=source,
+        title=run_title(run.playbook_path, labels.get(run.inventory_scope, run.inventory_scope)),
     )
 
 
@@ -300,14 +362,20 @@ async def run_detail(run_id: str):
         run = result.scalar_one_or_none()
         if run is None:
             return "Not found", 404
-        # NULL triggered_by = an automated (system) run; otherwise resolve the user.
-        triggered_label = "system"
-        if run.triggered_by:
-            res = await db.execute(
-                select(User.username).where(User.id == run.triggered_by))
-            triggered_label = res.scalar_one_or_none() or "(deleted user)"
+        origin = (await run_origins(db, [run]))[run.id]
+        labels = await scope_labels(db, [run.inventory_scope])
     return await render_template(
-        "ansible/run_detail.html", run=run, triggered_label=triggered_label)
+        "ansible/run_detail.html", run=run, origin=origin, hosts=hosts_worst_first(run.results),
+        row=run_view(run, labels, datetime.now(timezone.utc)))
+
+
+def hosts_worst_first(results: dict | None) -> list[tuple[str, dict]]:
+    """A run's per-host recap, failed or unreachable hosts first, then those
+    that changed something, then the rest, each by name."""
+    hosts = (results or {}).get("hosts") or {}
+    return sorted(hosts.items(), key=lambda kv: (
+        -((kv[1].get("failed") or 0) + (kv[1].get("unreachable") or 0) > 0),
+        -((kv[1].get("changed") or 0) > 0), kv[0]))
 
 
 @ansible_bp.post("/runs/<run_id>/cancel")
@@ -320,6 +388,8 @@ async def cancel_run(run_id: str):
         return "Not found", 404
     if run.status in (AnsibleRunStatus.running, AnsibleRunStatus.queued):
         executor.cancel_run(run_id)
+        await _audit("ansible.run.cancelled", "ansible_run", run_id,
+                     {"playbook": run.playbook_path})
     return redirect(url_for("ansible.run_detail", run_id=run_id))
 
 
@@ -368,6 +438,7 @@ async def schedules():
             r.id: r for r in (await db.execute(
                 select(AnsibleRun).where(AnsibleRun.id.in_(run_ids)))).scalars().all()
         } if run_ids else {}
+        labels = await scope_labels(db, [s.inventory_scope for s in scheds])
 
     source_data = [
         {"name": s["name"], "playbooks": src_module.discover_playbooks(s["path"])}
@@ -380,10 +451,12 @@ async def schedules():
         rows.append({
             "s": s,
             "last_run": runs.get(s.last_run_id) if s.last_run_id else None,
+            "scope_label": labels.get(s.inventory_scope, s.inventory_scope),
             "next_run": (s.last_run_at + timedelta(seconds=s.interval_seconds)) if s.last_run_at else None,
         })
         if editing_id and s.id == editing_id:
             editing = s
+    rows.sort(key=_schedule_order)
 
     # Default the source dropdown to the first source (or the edited schedule's
     # source), and pre-render that source's playbooks so the playbook dropdown
@@ -413,6 +486,23 @@ async def schedules():
     )
 
 
+def _schedule_order(row: dict) -> tuple:
+    """Schedules list order: failing first (its last run failed or was cut
+    short, or its last firing could not start one), then running, then the
+    rest, paused last; by name within each."""
+    s, run = row["s"], row["last_run"]
+    status = run.status.value if run else None
+    if not s.enabled:
+        rank = 3
+    elif status in ("failed", "interrupted") or (s.last_error and run is None):
+        rank = 0
+    elif status in ("queued", "running"):
+        rank = 1
+    else:
+        rank = 2
+    return rank, s.name.lower()
+
+
 def _schedule_form_fields(form) -> tuple[dict | None, str | None]:
     """Validate + extract schedule fields from a form. Returns (fields, error)."""
     name = (form.get("name", "") or "").strip()
@@ -436,6 +526,15 @@ def _schedule_form_fields(form) -> tuple[dict | None, str | None]:
     }, None
 
 
+def _schedule_detail(fields: dict) -> dict:
+    """What a schedule audit row records: what runs, where, how often — and
+    which variables, by name only."""
+    return {"name": fields["name"], "source": fields["source_name"],
+            "playbook": fields["playbook_path"], "scope": fields["inventory_scope"],
+            "interval_seconds": fields["interval_seconds"],
+            "vars": sorted(_run_vars(fields["params"]))}
+
+
 @ansible_bp.post("/schedules")
 @require_role(UserRole.operator)
 async def create_schedule():
@@ -443,9 +542,14 @@ async def create_schedule():
     fields, err = _schedule_form_fields(form)
     if err:
         return err, 400
+    err = guards.check_vars_for_role(_run_vars(fields["params"]), _caller_role())
+    if err:
+        return err, 400
+    sched = AnsibleSchedule(enabled=True, **fields)
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
-            db.add(AnsibleSchedule(enabled=True, **fields))
+            db.add(sched)
+    await _audit("ansible.schedule.created", "ansible_schedule", sched.id, _schedule_detail(fields))
     return redirect(url_for("ansible.schedules"))
 
 
@@ -461,8 +565,14 @@ async def update_schedule(schedule_id: str):
             sched = await db.get(AnsibleSchedule, schedule_id)
             if sched is None:
                 return "Not found", 404
+            err = guards.check_vars_for_role(
+                guards.changed_vars(_run_vars(fields["params"]), _run_vars(sched.params)),
+                _caller_role())
+            if err:
+                return err, 400
             for k, v in fields.items():
                 setattr(sched, k, v)
+    await _audit("ansible.schedule.updated", "ansible_schedule", schedule_id, _schedule_detail(fields))
     return redirect(url_for("ansible.schedules"))
 
 
@@ -475,6 +585,9 @@ async def toggle_schedule(schedule_id: str):
             if sched is None:
                 return "Not found", 404
             sched.enabled = not sched.enabled
+            enabled, name = sched.enabled, sched.name
+    await _audit("ansible.schedule.enabled" if enabled else "ansible.schedule.disabled",
+                 "ansible_schedule", schedule_id, {"name": name})
     return redirect(url_for("ansible.schedules"))
 
 
@@ -484,8 +597,11 @@ async def delete_schedule(schedule_id: str):
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
             sched = await db.get(AnsibleSchedule, schedule_id)
-            if sched is not None:
-                await db.delete(sched)
+            if sched is None:
+                return redirect(url_for("ansible.schedules"))
+            name = sched.name
+            await db.delete(sched)
+    await _audit("ansible.schedule.deleted", "ansible_schedule", schedule_id, {"name": name})
     return redirect(url_for("ansible.schedules"))
 
 
@@ -505,7 +621,7 @@ async def run_schedule_now(schedule_id: str):
         current_app._get_current_object(),  # type: ignore[attr-defined]
         source_name=source_name, playbook_path=playbook_path,
         inventory_scope=inventory_scope, params=params,
-        triggered_by=session["user_id"],
+        triggered_by=session["user_id"], origin="schedule", origin_id=schedule_id,
     )
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
@@ -514,6 +630,8 @@ async def run_schedule_now(schedule_id: str):
                 fresh.last_run_at = datetime.now(timezone.utc)
                 fresh.last_run_id = run.id if run else None
                 fresh.last_error = err
+    await _audit("ansible.schedule.run_now", "ansible_schedule", schedule_id,
+                 {"playbook": playbook_path, "run_id": run.id if run else None, "error": err})
     if run:
         return redirect(url_for("ansible.run_detail", run_id=run.id))
     return redirect(url_for("ansible.schedules"))
@@ -590,6 +708,9 @@ async def save_playbook():
             content=content, error=err,
         ), 400
 
+    await _audit("ansible.playbook.updated" if editing else "ansible.playbook.created",
+                 "ansible_playbook", f"{source_name}:{playbook_path}",
+                 {"source": source_name, "playbook": playbook_path, "bytes": len(content)})
     return redirect(url_for(
         "ansible.view_playbook", source_name=source_name, playbook_path=playbook_path))
 
@@ -601,9 +722,12 @@ async def delete_playbook():
     source = _editable_source((form.get("source_name", "") or "").strip())
     if source is None:
         return "Source is not editable", 400
-    ok, err = src_module.delete_playbook(source["path"], (form.get("playbook_path", "") or "").strip())
+    playbook_path = (form.get("playbook_path", "") or "").strip()
+    ok, err = src_module.delete_playbook(source["path"], playbook_path)
     if not ok:
         return err, 400
+    await _audit("ansible.playbook.deleted", "ansible_playbook", f"{source['name']}:{playbook_path}",
+                 {"source": source["name"], "playbook": playbook_path})
     return redirect(url_for("ansible.browse"))
 
 

steward/ansible/runner.py

@@ -10,12 +10,25 @@ import asyncio
 import uuid
 from datetime import datetime, timezone
 
-from steward.ansible import executor, sources as src_module
+from steward.ansible import executor, guards, sources as src_module
 from steward.ansible.inventory_gen import (
     fetch_scope_targets, generate_inventory, inventory_to_yaml,
 )
 from steward.models.ansible import AnsibleRun, AnsibleRunStatus
 
+# A run in one of these hasn't finished: a schedule or an alert action whose
+# previous run is still in one skips its next firing (#5158).
+IN_FLIGHT = (AnsibleRunStatus.queued, AnsibleRunStatus.running)
+
+
+async def run_in_flight(db, run_id: str | None) -> bool:
+    """Whether run_id names a run that hasn't finished. Runs left running by a
+    restart are marked interrupted at startup, so this can't stick."""
+    if not run_id:
+        return False
+    run = await db.get(AnsibleRun, run_id)
+    return run is not None and run.status in IN_FLIGHT
+
 
 async def trigger_run(
     app,
@@ -25,13 +38,18 @@ async def trigger_run(
     inventory_scope: str = "steward:all",
     params: dict | None = None,
     triggered_by: str | None = None,
+    origin: str | None = None,
+    origin_id: str | None = None,
     inventory_content: str | None = None,
     connection: dict | None = None,
     secret_vars: dict | None = None,
+    redact: list[str] | None = None,
 ):
     """Resolve inventory for the scope, persist an AnsibleRun, and launch it.
 
     triggered_by=None marks a system/automated run (alerts, schedules).
+    origin / origin_id say where it came from (AnsibleRun.origin), so the runs
+    list can say "schedule nightly" or "alert rule nas down".
     If inventory_content is provided, it is used verbatim and scope resolution
     is skipped (the caller built a bespoke inventory — e.g. host_agent deploy
     injecting per-host tokens); inventory_scope is still recorded for display.
@@ -39,7 +57,8 @@ async def trigger_run(
     first-contact provisioning — passed to the executor but deliberately NOT
     stored on the AnsibleRun row (params), so the password never lands in the DB.
     secret_vars (sensitive run-time playbook variables) are likewise passed
-    through to the executor but never persisted.
+    through to the executor but never persisted. redact names further values
+    to mask in the run's output (a bespoke inventory's per-host tokens).
     Returns (run, source, error): on success error is None; on failure run is
     None and error is a short human-readable reason.
     """
@@ -47,6 +66,13 @@ async def trigger_run(
     source = next((s for s in sources if s["name"] == source_name), None)
     if source is None:
         return None, None, "Source not found"
+    # Every run path (web, MCP, alerts, schedules, plugins) arrives here, so
+    # this is where a run is held to what its source lists (ansible/guards.py).
+    err = guards.check_playbook(source, playbook_path)
+    if err is None and inventory_content is None:
+        err = guards.check_inventory_scope(source, inventory_scope)
+    if err:
+        return None, source, err
 
     inventory_path: str | None = None
     if inventory_content is not None:
@@ -55,11 +81,8 @@ async def trigger_run(
         async with app.db_sessionmaker() as db:
             targets = await fetch_scope_targets(db, inventory_scope)
         inventory_content = inventory_to_yaml(generate_inventory(targets))
-    elif inventory_scope.startswith("repo:"):
-        parts = inventory_scope.split(":", 2)
-        inventory_path = parts[2] if len(parts) == 3 else ""
     else:
-        return None, source, "Invalid inventory_scope"
+        inventory_path = inventory_scope.split(":", 2)[2]
 
     run_id = str(uuid.uuid4())
     run = AnsibleRun(
@@ -69,6 +92,8 @@ async def trigger_run(
         inventory_scope=inventory_scope,
         source_name=source_name,
         triggered_by=triggered_by,
+        origin=origin,
+        origin_id=origin_id,
         status=AnsibleRunStatus.running,
         started_at=datetime.now(timezone.utc),
         params=params or None,
@@ -81,7 +106,7 @@ async def trigger_run(
         executor.start_run(
             app, run_id, playbook_path, inventory_path or "",
             source["path"], params or None, inventory_content,
-            connection=connection, secret_vars=secret_vars,
+            connection=connection, secret_vars=secret_vars, redact=redact,
         )
     )
     task.add_done_callback(

steward/ansible/scheduler.py

@@ -7,7 +7,7 @@ from datetime import datetime, timezone
 
 from sqlalchemy import select
 
-from steward.ansible.runner import trigger_run
+from steward.ansible.runner import run_in_flight, trigger_run
 from steward.models.ansible_schedule import AnsibleSchedule
 
 logger = logging.getLogger(__name__)
@@ -27,7 +27,8 @@ def is_due(schedule: AnsibleSchedule, now: datetime) -> bool:
 
 
 async def run_due_schedules(app) -> int:
-    """Trigger every due schedule once. Returns the number launched."""
+    """Trigger every due schedule once. Returns the number launched. A due
+    schedule whose previous run hasn't finished skips this firing."""
     now = datetime.now(timezone.utc)
     async with app.db_sessionmaker() as db:
         schedules = (await db.execute(
@@ -38,6 +39,21 @@ async def run_due_schedules(app) -> int:
     for s in schedules:
         if not is_due(s, now):
             continue
+        async with app.db_sessionmaker() as db:
+            busy = await run_in_flight(db, s.last_run_id)
+        if busy:
+            # One run at a time per schedule (#5158). This firing is skipped,
+            # not deferred, so the cadence holds; last_run_id still names the
+            # run that's going.
+            async with app.db_sessionmaker() as db:
+                async with db.begin():
+                    fresh = await db.get(AnsibleSchedule, s.id)
+                    if fresh:
+                        fresh.last_run_at = now
+                        fresh.last_error = "Skipped: the previous run was still going"
+            logger.info("Schedule %s (%s) skipped: run %s still going",
+                        s.id, s.name, s.last_run_id)
+            continue
         run = None
         err: str | None = None
         try:
@@ -48,6 +64,8 @@ async def run_due_schedules(app) -> int:
                 inventory_scope=s.inventory_scope,
                 params=s.params,
                 triggered_by=None,
+                origin="schedule",
+                origin_id=s.id,
             )
         except Exception:
             logger.exception("Schedule %s (%s) failed to launch", s.id, s.name)

steward/ansible/sources.py

@@ -3,8 +3,10 @@ import asyncio
 import logging
 import os
 import re
+import shutil
 import stat
 import tempfile
+import time
 from pathlib import Path
 
 logger = logging.getLogger(__name__)
@@ -76,6 +78,9 @@ BUILTIN_SOURCE_NAME = "steward-builtin"
 # persistent /data volume; created lazily on first save.
 LOCAL_SOURCE_NAME = "steward-local"
 USER_PLAYBOOK_DIR = os.environ.get("STEWARD_PLAYBOOK_DIR", "/data/ansible/playbooks")
+# Git sources are checked out here, on the persistent /data volume, so a
+# redeploy keeps them instead of re-cloning into the container's layer (#5157).
+SOURCE_CACHE_DIR = os.environ.get("STEWARD_ANSIBLE_SOURCE_DIR", "/data/ansible/sources")
 
 
 def _builtin_source() -> dict:
@@ -108,6 +113,13 @@ def is_editable_source(source: dict) -> bool:
     return source.get("type") == "local" and source.get("name") != BUILTIN_SOURCE_NAME
 
 
+def _open_token(src: dict) -> str:
+    """The source's git token in the clear (it is stored sealed)."""
+    from steward.core.crypto import decrypt_secret
+    from steward.core.settings import source_token_key
+    return decrypt_secret(src.get("http_token") or "", context=source_token_key(src.get("name", "?")))
+
+
 def get_sources(ansible_cfg: dict) -> list[dict]:
     """Return resolved source list from ansible config section.
 
@@ -117,7 +129,7 @@ def get_sources(ansible_cfg: dict) -> list[dict]:
     Config structure in config.yaml::
 
         ansible:
-          cache_dir: /var/cache/steward/ansible
+          cache_dir: /data/ansible/sources
           sources:
             - name: my-playbooks
               type: local
@@ -129,7 +141,7 @@ def get_sources(ansible_cfg: dict) -> list[dict]:
               pull_interval_seconds: 3600
     """
     sources = ansible_cfg.get("sources", [])
-    cache_dir = ansible_cfg.get("cache_dir", "/var/cache/steward/ansible")
+    cache_dir = ansible_cfg.get("cache_dir") or SOURCE_CACHE_DIR
     # Always expose the bundled (read-only) + local (writable) first-party sources.
     result = [_builtin_source(), _local_source()]
     for src in sources:
@@ -149,7 +161,7 @@ def get_sources(ansible_cfg: dict) -> list[dict]:
             "url": src.get("url"),
             "branch": src.get("branch", "main"),
             "pull_interval_seconds": int(src.get("pull_interval_seconds", 3600)),
-            "http_token": src.get("http_token", "") if src_type == "git" else "",
+            "http_token": _open_token(src) if src_type == "git" else "",
         })
     return result
 
@@ -323,66 +335,117 @@ def validate_playbook_yaml(content: str) -> tuple[bool, str | None]:
     return True, None
 
 
-async def git_pull(source: dict) -> None:
-    """Clone the git repo if absent; pull if already present.
+# The askpass script holds no secret: git runs it with the token in its
+# environment, and printf prints it as data, so no character in a token can
+# become shell syntax.
+ASKPASS_TOKEN_ENV = "STEWARD_GIT_TOKEN"
+ASKPASS_SCRIPT = (
+    "#!/bin/sh\n"
+    'case "$1" in\n'
+    "  Username*) printf '%s\\n' oauth2 ;;\n"
+    f"  Password*) printf '%s\\n' \"${ASKPASS_TOKEN_ENV}\" ;;\n"
+    "esac\n"
+)
 
-    When source['http_token'] is set, injects credentials via a temporary
-    GIT_ASKPASS script so they never touch .git/config or process args.
+
+# A clone or pull past this is cut, so a hung remote can't hold a source's
+# pull (or the Sync button) open forever (#5157).
+GIT_TIMEOUT_SECONDS = 120.0
+
+
+class GitPullError(RuntimeError):
+    """A clone or pull that failed or ran past its timeout; says which and why."""
+
+
+async def _git(argv: list[str], env: dict, timeout: float, what: str) -> None:
+    from steward.ansible.executor import _terminate_then_kill
+
+    proc = await asyncio.create_subprocess_exec(
+        *argv,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+        env=env,
+        start_new_session=True,   # its own group, so a timeout ends ssh/helpers too
+    )
+    try:
+        _, stderr = await asyncio.wait_for(proc.communicate(), timeout)
+    except asyncio.TimeoutError:
+        raise GitPullError(f"{what} did not finish within {timeout:g} s") from None
+    finally:
+        if proc.returncode is None:
+            await asyncio.shield(_terminate_then_kill(proc, grace=2.0))
+    if proc.returncode != 0:
+        raise GitPullError(f"{what} failed: {stderr.decode(errors='replace').strip()[-500:]}")
+
+
+async def git_pull(source: dict, timeout: float = GIT_TIMEOUT_SECONDS) -> None:
+    """Clone the git repo if absent; fast-forward it if already present.
+
+    Raises GitPullError when git fails or runs past `timeout`; a clone that
+    did not finish is removed, so the next attempt clones afresh rather than
+    pulling into a broken checkout. git never prompts (GIT_TERMINAL_PROMPT=0):
+    a missing credential fails at once instead of waiting on input.
+
+    When source['http_token'] is set, git asks a temporary GIT_ASKPASS script,
+    which prints the token from the child's environment, so it never touches
+    .git/config, process args or a file.
     """
     path = Path(source["path"])
     token = source.get("http_token", "")
 
-    env = None
+    env = {**os.environ, "GIT_TERMINAL_PROMPT": "0"}
     askpass_path = None
     if token:
         fd, askpass_path = tempfile.mkstemp(prefix="steward-askpass-", suffix=".sh")
         try:
-            script = (
-                "#!/bin/sh\n"
-                'case "$1" in\n'
-                '  Username*) echo "oauth2" ;;\n'
-                f'  Password*) echo "{token}" ;;\n'
-                "esac\n"
-            )
-            os.write(fd, script.encode())
+            os.write(fd, ASKPASS_SCRIPT.encode())
         finally:
             os.close(fd)
         os.chmod(askpass_path, stat.S_IRWXU)
-        env = {**os.environ, "GIT_ASKPASS": askpass_path}
+        env.update({"GIT_ASKPASS": askpass_path, ASKPASS_TOKEN_ENV: token})
 
     try:
         if not (path / ".git").exists():
             path.mkdir(parents=True, exist_ok=True)
-            proc = await asyncio.create_subprocess_exec(
-                "git", "clone",
-                "--branch", source["branch"],
-                "--single-branch",
-                source["url"], str(path),
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-                env=env,
-            )
-            _, stderr = await proc.communicate()
-            if proc.returncode != 0:
-                logger.error(
-                    "git clone failed for %r: %s",
-                    source["name"],
-                    stderr.decode(errors="replace"),
-                )
+            try:
+                await _git(["git", "clone", "--branch", source["branch"], "--single-branch",
+                            source["url"], str(path)],
+                           env, timeout, f"git clone of {source['name']!r}")
+            except BaseException:
+                shutil.rmtree(path, ignore_errors=True)
+                raise
         else:
-            proc = await asyncio.create_subprocess_exec(
-                "git", "-C", str(path), "pull",
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-                env=env,
-            )
-            _, stderr = await proc.communicate()
-            if proc.returncode != 0:
-                logger.error(
-                    "git pull failed for %r: %s",
-                    source["name"],
-                    stderr.decode(errors="replace"),
-                )
+            await _git(["git", "-C", str(path), "pull", "--ff-only"],
+                       env, timeout, f"git pull of {source['name']!r}")
     finally:
         if askpass_path and os.path.exists(askpass_path):
             os.unlink(askpass_path)
+
+
+# When each git source was last pulled (monotonic seconds), by name.
+_last_pull: dict[str, float] = {}
+
+
+async def pull_due_sources(ansible_cfg: dict, now: float | None = None) -> list[str]:
+    """Pull every git source whose pull_interval_seconds has passed.
+
+    Run by one scheduled task on a short tick, reading the sources as they are
+    now, so a source added in Settings is pulled within a tick and an edited
+    interval applies without a restart (#5157). A failed pull is logged and
+    waits out its interval like a good one. Returns the names pulled.
+    """
+    now = time.monotonic() if now is None else now
+    sources = get_sources(ansible_cfg)
+    due = [src for src in sources
+           if src["type"] == "git"
+           and now - _last_pull.get(src["name"], float("-inf")) >= src["pull_interval_seconds"]]
+    names = {src["name"] for src in sources}
+    for gone in [n for n in _last_pull if n not in names]:
+        del _last_pull[gone]
+    for src in due:
+        _last_pull[src["name"]] = now
+        try:
+            await git_pull(src)
+        except Exception:
+            logger.exception("Ansible source %r: pull failed", src["name"])
+    return [src["name"] for src in due]

steward/app.py

@@ -3,9 +3,12 @@ from __future__ import annotations
 import asyncio
 import logging
 from pathlib import Path
-from quart import Quart, render_template
+from quart import Quart, g, render_template, request
 from .config import load_bootstrap
-from .database import init_db
+from .database import (
+    init_db, ensure_database_reachable, DatabaseUnavailable,
+    DB_CONNECT_TIMEOUT_SECONDS,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -23,6 +26,7 @@ def create_app(
         bootstrap = {
             "database_url": "postgresql+asyncpg://test/test",
             "secret_key": "test-secret-key",
+            "db_connect_timeout": DB_CONNECT_TIMEOUT_SECONDS,
             "plugin_dirs": ["plugins"],
             "plugin_install_dir": "plugins",
         }
@@ -33,6 +37,9 @@ def create_app(
         PLUGIN_DIRS=bootstrap["plugin_dirs"],
         PLUGIN_INSTALL_DIR=bootstrap["plugin_install_dir"],
         TESTING=testing,
+        # Lax keeps the session cookie off cross-site POSTs and iframes; the
+        # CSRF token below is the check that does not depend on the browser.
+        SESSION_COOKIE_SAMESITE="Lax",
     )
 
     # ── 2. Init DB engine ──────────────────────────────────────────────────────
@@ -42,6 +49,19 @@ def create_app(
         from unittest.mock import MagicMock
         app.db_sessionmaker = MagicMock()
 
+    # ── 2b. Block until the database is actually reachable ────────────────────
+    # Everything from here down touches the DB synchronously (migrations,
+    # secret re-encryption, settings load). Gate all of it behind one readiness
+    # check so a DB that is merely slow to come up — WAL recovery after an
+    # unclean shutdown, or Docker DNS not yet serving the `db` name after a host
+    # reboot — is waited out instead of crashing the container on a traceback.
+    if not testing:
+        from .database import wait_for_database
+        wait_for_database(
+            app.config["DATABASE_URL"],
+            timeout_seconds=bootstrap["db_connect_timeout"],
+        )
+
     # ── 3. Core migrations only (creates app_settings table) ──────────────────
     if not testing:
         from .core.migration_runner import run_core_migrations
@@ -70,13 +90,16 @@ def create_app(
         from .core.settings import (
             load_settings_sync, to_smtp_cfg, to_webhook_cfg,
             to_ansible_cfg, to_plugins_cfg, to_oidc_cfg, to_ldap_cfg,
-            to_thresholds_cfg,
+            to_thresholds_cfg, to_access_cfg,
         )
         _settings = load_settings_sync(app.config["DATABASE_URL"])
+        from .core.logging_setup import set_log_level
+        set_log_level(_settings.get("log.level"))
+        app.config.update(to_access_cfg(_settings))
         app.config.update(
-            SESSION_LIFETIME_HOURS=_settings.get("session.lifetime_hours", 8),
             DATA_RETENTION_DAYS=_settings.get("data.retention_days", 90),
             MONITORS_POLL_INTERVAL=_settings.get("monitors.poll_interval_seconds", 60),
+            PUBLIC_BASE_URL=_settings.get("general.public_base_url", ""),
             SMTP=to_smtp_cfg(_settings),
             WEBHOOK=to_webhook_cfg(_settings),
             ANSIBLE=to_ansible_cfg(_settings),
@@ -86,11 +109,12 @@ def create_app(
             THRESHOLDS=to_thresholds_cfg(_settings),
         )
     else:
-        from .core.settings import to_thresholds_cfg
+        from .core.settings import to_access_cfg, to_thresholds_cfg
+        app.config.update(to_access_cfg({}))
         app.config.update(
-            SESSION_LIFETIME_HOURS=8,
             DATA_RETENTION_DAYS=90,
             MONITORS_POLL_INTERVAL=60,
+            PUBLIC_BASE_URL="",
             SMTP={},
             WEBHOOK={},
             ANSIBLE={},
@@ -108,6 +132,59 @@ def create_app(
         lambda value, kind: _ts_for(value, kind, app.config.get("THRESHOLDS", {}))
     )
 
+    # One way to write a size, a rate, a span and a moment, and to grade a
+    # reading ('ok' / 'warn' / 'crit' / 'unknown') against the same cutoffs.
+    from .core import formatting as _fmt
+    app.jinja_env.filters["bytes"] = _fmt.human_bytes
+    app.jinja_env.filters["rate"] = _fmt.human_rate
+    app.jinja_env.filters["duration"] = _fmt.human_duration
+    app.jinja_env.filters["ago"] = _fmt.ago
+    app.jinja_env.globals["status_class"] = (
+        lambda value, kind: _fmt.status_class(value, kind, app.config.get("THRESHOLDS", {}))
+    )
+    # The status and layout macros in _macros.html read these.
+    app.jinja_env.globals["status_info"] = _fmt.status_info
+    app.jinja_env.globals["required"] = _fmt.required
+    # A metric and its source in words, a reading in its unit (alerts pages).
+    for _name in ("metric_label", "metric_about", "metric_unit", "metric_value", "source_label",
+                  "cert_left", "field_label"):
+        app.jinja_env.globals[_name] = getattr(_fmt, _name)
+    from .alerts.service import rule_condition as _rule_condition
+    app.jinja_env.globals["rule_condition"] = _rule_condition
+    from .models.monitors import MONITOR_TYPE_LABELS as _type_labels
+    app.jinja_env.filters["monitor_type"] = lambda t: _type_labels.get(t, str(t).upper())
+
+    # Forged-request protection: every POST form renders csrf_field(), HTMX and
+    # fetch() send csrf_token() as X-CSRF-Token, and one check refuses any
+    # unsafe request without it (steward/auth/csrf.py).
+    from .auth import csrf as _csrf
+    app.jinja_env.globals["csrf_token"] = _csrf.csrf_token
+    app.jinja_env.globals["csrf_field"] = _csrf.csrf_field
+    app.before_request(_csrf.check_request)
+    # The session cookie is Secure, and HSTS is sent, only on requests that
+    # arrived over HTTPS — directly or as a trusted proxy reports it
+    # (steward/core/proxy.py). Plain-HTTP and LAN use keep working.
+    from quart import has_request_context
+    from quart.sessions import SecureCookieSessionInterface
+    from .core.proxy import is_https
+
+    class _SessionInterface(SecureCookieSessionInterface):
+        def get_cookie_secure(self, app):
+            return bool(app.config.get("SESSION_COOKIE_SECURE")) or (
+                has_request_context() and is_https())
+
+    app.session_interface = _SessionInterface()
+
+    @app.after_request
+    async def _hsts(response):
+        if is_https():
+            response.headers.setdefault("Strict-Transport-Security", "max-age=31536000")
+        return response
+
+    # Default-deny: a view with neither require_role nor public_route is refused.
+    from .auth.middleware import check_view_is_marked, public_route
+    app.before_request(check_view_is_marked)
+
     # ── 5. Plugin migrations ───────────────────────────────────────────────────
     # Step 3 already applied all plugin migrations via discover_all_plugin_migration_dirs,
     # so this is a no-op on normal startup. We still run it with all discovered dirs so
@@ -133,6 +210,7 @@ def create_app(
     from .ansible.inventory_routes import inventory_bp
     from .settings.routes import settings_bp
     from .audit.routes import audit_bp
+    from .ai_access.routes import ai_access_bp
 
     app.register_blueprint(auth_bp)
     app.register_blueprint(dashboard_bp)
@@ -144,6 +222,7 @@ def create_app(
     app.register_blueprint(inventory_bp)
     app.register_blueprint(settings_bp)
     app.register_blueprint(audit_bp)
+    app.register_blueprint(ai_access_bp)
 
     # Register the unified Monitor status source for the Status page. Plugins
     # may register additional sources from setup(). Idempotent.
@@ -198,39 +277,71 @@ def create_app(
             "undecryptable_secrets": get_undecryptable_secrets(),
         }
 
+    @app.context_processor
+    async def _inject_alert_count():
+        """The sidebar's count of alerts needing attention. Full pages only:
+        an htmx fragment has no sidebar, and a logged-out page has no nav."""
+        from quart import session
+        if (request.headers.get("HX-Request") or not session.get("user_id")
+                or g.get("database_down")):
+            return {}
+        try:
+            from .alerts.service import active_count
+            async with app.db_sessionmaker() as db:
+                return {"active_alert_count": await active_count(db)}
+        except Exception:
+            # A count is decoration; the page renders without it.
+            logging.getLogger(__name__).exception("Active alert count failed")
+            return {}
+
     # ── 11. Share-token middleware ─────────────────────────────────────────────
     @app.before_request
     async def _validate_share_token():
-        """If ?s=<token> is present, validate it and set g.share_viewer."""
+        """A share link's token (?s=) opens, read-only, exactly the widget
+        fragments its own dashboard loads (steward/dashboard/share.py)."""
         from quart import request, g
         raw = request.args.get("s")
-        if not raw:
+        if not raw or request.method != "GET":
             return
         try:
-            import hashlib
-            from datetime import datetime, timezone
-            from sqlalchemy import select
-            from .models.dashboard import DashboardShareToken
-            token_hash = hashlib.sha256(raw.encode()).hexdigest()
+            from .dashboard.share import share_grants
             async with app.db_sessionmaker() as db:
-                result = await db.execute(
-                    select(DashboardShareToken)
-                    .where(DashboardShareToken.token_hash == token_hash)
-                )
-                share = result.scalar_one_or_none()
-            if share:
-                now = datetime.now(timezone.utc)
-                if share.expires_at is None or share.expires_at > now:
+                if await share_grants(db, raw, request.path, request.args):
                     g.share_viewer = True
-                    g.share_dashboard_id = share.dashboard_id
         except Exception:
-            pass  # never block a request due to token validation failure
+            # A failed check grants nothing; the request falls through to login.
+            logging.getLogger(__name__).exception("Share token check failed")
 
     # ── 11. Health check ───────────────────────────────────────────────────────
     @app.get("/health")
+    @public_route("liveness probe for the container")
     async def health():
         return {"status": "ok"}
 
+    # ── 11b. Database availability gate ────────────────────────────────────────
+    # Every page in Steward reads the database, so a database that has gone away
+    # under a running app turns each request into an opaque 500 (this is how a
+    # login attempt surfaced a bare gaierror). Acquire a connection up front,
+    # retrying briefly to ride out a restart, and answer honestly if it stays
+    # down rather than failing deep inside a handler with a generic error.
+    if not testing:
+        @app.before_request
+        async def _database_gate():
+            # /health is a liveness probe for the container itself — it must
+            # stay answerable while the database is down, or a restart loop
+            # gets triggered by a dependency outage. Static files need no DB.
+            if request.endpoint in ("health", "static"):
+                return None
+            try:
+                await ensure_database_reachable(app._db_engine)
+            except DatabaseUnavailable as exc:
+                logger.error("Database unreachable while serving %s: %s",
+                             request.path, exc)
+                g.database_down = True
+                return await render_template(
+                    "errors/database_unavailable.html"), 503
+            return None
+
     # ── 12. Error handlers ─────────────────────────────────────────────────────
     @app.errorhandler(404)
     async def not_found(_):
@@ -240,20 +351,40 @@ def create_app(
     async def server_error(_):
         return await render_template("errors/500.html"), 500
 
+    # ── Session freshness: lifetime, deactivation and role changes apply per request
+    if not testing:
+        from .auth.middleware import refresh_session_user
+        app.before_request(refresh_session_user)
+
+    # ── MCP endpoint (/mcp) ───────────────────────────────────────────────────
+    # Mounted after plugins load so their register_mcp_tools() hooks are seen.
+    # It wraps app.asgi_app: /mcp never enters Quart's pipeline (see mcp/server.py).
+    from .mcp import mount_mcp
+    mount_mcp(app)
+
     # ── 11. Startup hook ───────────────────────────────────────────────────────
     @app.before_serving
     async def startup():
         if not testing:
             await _mark_interrupted_runs(app)
             asyncio.create_task(_run_scheduler(app))
+            app._latest_backfill = asyncio.create_task(_backfill_latest(app))
 
     return app
 
 
+async def _backfill_latest(app: Quart) -> None:
+    from .core.metric_series import backfill_latest
+    try:
+        await backfill_latest(app.db_sessionmaker, app.logger)
+    except Exception:
+        app.logger.exception("latest metrics: backfill failed; quiet resources fill on their next sample")
+
+
 def _register_core_tasks(app: Quart) -> None:
     from .core.scheduler import ScheduledTask
 
-    poll_interval = app.config.get("MONITORS_POLL_INTERVAL", 60)
+    from .monitors.scheduler import DISPATCH_TICK_SECONDS as MONITOR_DISPATCH_TICK
     cleanup_interval = 3600  # hourly
 
     async def run_monitor_checks():
@@ -264,6 +395,14 @@ def _register_core_tasks(app: Quart) -> None:
         from .core.cleanup import run_cleanup as _cleanup
         await _cleanup(app)
 
+    async def run_alert_reconcile():
+        from .core.alerts import check_silence, notify_after_maintenance
+        for step in (check_silence, notify_after_maintenance):
+            try:
+                await step(app)
+            except Exception:
+                app.logger.exception("alert_reconcile: %s failed", step.__name__)
+
     async def run_report_check():
         from .core.reports import check_and_send
         await check_and_send(app)
@@ -288,10 +427,20 @@ def _register_core_tasks(app: Quart) -> None:
             interval_seconds=60,
             run_on_startup=True,
         ),
+        # Rules whose series has gone quiet get a visible no_data mark; alerts
+        # a maintenance window muted notify once it ends.
+        ScheduledTask(
+            name="alert_reconcile",
+            coro_factory=run_alert_reconcile,
+            interval_seconds=60,
+            run_on_startup=False,
+        ),
+        # Ticks often and only starts checks: each monitor decides whether it
+        # is due from its own interval (steward/monitors/scheduler.py).
         ScheduledTask(
             name="monitor_check",
             coro_factory=run_monitor_checks,
-            interval_seconds=poll_interval,
+            interval_seconds=MONITOR_DISPATCH_TICK,
             run_on_startup=True,
         ),
         ScheduledTask(
@@ -302,26 +451,21 @@ def _register_core_tasks(app: Quart) -> None:
         ),
     ])
 
-    # Register git-pull tasks for each ansible git source
-    ansible_cfg = app.config.get("ANSIBLE", {})
-    from .ansible.sources import get_sources
-    for source in get_sources(ansible_cfg):
-        if source["type"] != "git":
-            continue
-        _source = source
+    # Pull Ansible git sources as each falls due. One task reads the sources
+    # as they are on every tick, so one added in Settings is pulled within
+    # half a minute, no restart (#5157).
+    async def run_git_pulls():
+        from .ansible.sources import pull_due_sources
+        await pull_due_sources(app.config.get("ANSIBLE", {}))
 
-        async def run_git_pull(src=_source):
-            from .ansible.sources import git_pull
-            await git_pull(src)
-
-        app._task_registry.append(
-            ScheduledTask(
-                name=f"ansible_git_pull_{_source['name']}",
-                coro_factory=run_git_pull,
-                interval_seconds=_source["pull_interval_seconds"],
-                run_on_startup=True,
-            )
+    app._task_registry.append(
+        ScheduledTask(
+            name="ansible_git_pulls",
+            coro_factory=run_git_pulls,
+            interval_seconds=30,
+            run_on_startup=True,
         )
+    )
 
     # Fire due Ansible schedules (recurring playbook runs). Checks every minute;
     # each schedule's own interval gates whether it actually runs.