docs(host-agent): document container-log collection [M79 step 6]

Design-spec updates for m79: the docker_logs_enabled / docker_log_exclude config
keys, the collect_docker_logs collector + the drop-logs-on-backoff note in the
agent internals, and a wire-format design point on docker_logs riding in the same
push (with server-side toggle/exclude enforcement + the per-container ring).

The end-to-end chain (store → rotate → query + ingest enforcement) is already
covered by the integration tests added across steps 2–5, so no capstone test is
added. AGENT_VERSION bump (1.7.0) landed in step 1.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CAGR73DUowdVFVvYzLXC5C

.forgejo/workflows/ci.yml

@@ -14,6 +14,17 @@ on:
   # pull_request intentionally absent — push on [dev, main] already fires CI for
   # every dev commit and dev→main PRs. Single-operator repo, no fork PRs.
 
+# Serialize CI runs per ref so the publish lane never shares the runner's docker
+# daemon with another run. Two publishes racing on one daemon evict each other's
+# freshly-built image mid-push, breaking the post-build tag/push (issue #1093).
+# cancel-in-progress:false is deliberate — rule 46 requires EVERY dev/main push to
+# publish its own immutable :<sha> image, so a superseded run must still run to
+# completion (never cancelled) to emit its SHA. (Forgejo honors `concurrency:` —
+# CI-runner's build workflows use it.)
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: false
+
 jobs:
   # Fast-fail lint lane. ruff is pre-installed in the ci-python image, so this
   # runs with NO dependency install and surfaces lint bounces in seconds.
@@ -115,14 +126,25 @@ jobs:
         run: |
           set -euxo pipefail
           IMAGE=git.fabledsword.com/bvandeusen/steward
+          # The moving tag for this ref: dev→:dev, main→:latest (rule 46). main IS
+          # the production line, so :latest tracks main's tip; there is no :main.
+          MOVING=""
+          if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
+            MOVING="dev"
+          elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
+            MOVING="latest"
+          fi
+          # Tag BOTH targets at build time (mirrors CI-runner's build workflows):
+          # one `docker build -t :<sha> -t :<moving>` points both tags at the image
+          # atomically, so we never run a post-push `docker tag` of an image a
+          # concurrent run could have evicted from the shared daemon (issue #1093).
+          if [ -n "$MOVING" ]; then
+            docker build -t "$IMAGE:${{ github.sha }}" -t "$IMAGE:$MOVING" .
+            docker push "$IMAGE:${{ github.sha }}"
+            docker push "$IMAGE:$MOVING"
+          else
             docker build -t "$IMAGE:${{ github.sha }}" .
             docker push "$IMAGE:${{ github.sha }}"
-          if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
-            docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:dev"
-            docker push "$IMAGE:dev"
-          elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
-            docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:latest"
-            docker push "$IMAGE:latest"
           fi
       - name: Prune dangling layers
         if: always()

Dockerfile

@@ -7,6 +7,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \
         gosu \
         # openssh-client — ansible-playbook reaches remote hosts over ssh
         openssh-client \
+        # sshpass — Ansible shells out to it for first-contact password SSH
+        # (provisioning a fresh host before the managed key is installed)
+        sshpass \
     && rm -rf /var/lib/apt/lists/*
 
 # Upgrade pip to suppress the version notice
@@ -16,8 +19,10 @@ WORKDIR /app
 
 COPY pyproject.toml .
 COPY steward/ steward/
-# .[ansible] pulls the full Ansible package so the playbook runner works in-image.
-RUN pip install --no-cache-dir '.[ansible]'
+# Bundle the extras whose first-party plugins ship in the image: [ansible] for
+# the playbook runner, [snmp] (pysnmp) for the SNMP poller. Without them those
+# bundled plugins load but silently no-op for want of their runtime dependency.
+RUN pip install --no-cache-dir '.[ansible,snmp]'
 
 COPY alembic.ini .
 # First-party plugins ship inside the image (bundled root at /app/plugins).

compose.deploy.yml

@@ -22,12 +22,22 @@ services:
       # /data holds the auto-generated secret key, third-party plugins, and the
       # Ansible playbook cache — all of which must survive image updates. No
       # source bind-mounts: core + first-party plugins live inside the image.
+      #
+      # The container runs as the unprivileged 'app' user (uid 1000), so /data
+      # MUST be writable by uid 1000. A named docker volume (below) gets that
+      # automatically. If you bind-mount a host/NFS path instead, chown it to
+      # 1000:1000 (and mind NFS root_squash) — otherwise the app can't persist
+      # its secret key, mints a fresh one each boot, and every encrypted secret
+      # (managed SSH key, SMTP/OIDC/LDAP creds) becomes unrecoverable. The app
+      # now refuses to start in that state rather than silently degrade.
       - app_data:/data
     environment:
       - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:${POSTGRES_PASSWORD:-steward}@db/steward
       # STEWARD_SECRET_KEY is intentionally absent — the app generates a random
       # key on first boot and persists it to /data/secret.key, which the app_data
-      # volume keeps across image updates.
+      # volume keeps across image updates. On multi-node Swarm with a shared bind
+      # mount, pin STEWARD_SECRET_KEY (env or a docker secret) instead, so the key
+      # never depends on filesystem ownership being correct on every node.
     depends_on:
       db:
         condition: service_healthy

docs/plugins/host-agent-design.md

@@ -74,8 +74,15 @@ token = a1b2c3d4...
 interval_seconds = 30
 hostname = myhost              # optional; defaults to uname -n
 mounts = /, /mnt/data          # optional; defaults to all real mounts (excluding tmpfs/devtmpfs/etc.)
+docker_logs_enabled = true     # optional; on by default — per-host container-log kill-switch
+docker_log_exclude = watchtower, noisy-svc   # optional; container names whose logs this host skips collecting
 ```
 
+Container-log collection is on by default (no config needed). The two keys above
+are the per-host opt-outs; the operator-facing global toggle + exclude live in
+Settings and are enforced server-side (the push model has no channel to command
+an agent), so they don't require touching a host's conf.
+
 ### Agent internals (function list, not classes)
 
 - `read_config(path)` — parses the conf file into a dict.
@@ -85,10 +92,17 @@ mounts = /, /mnt/data          # optional; defaults to all real mounts (excludin
 - `collect_load()` — reads `/proc/loadavg`, returns `[1m, 5m, 15m]`.
 - `collect_uptime()` — reads `/proc/uptime`, returns seconds since boot (int).
 - `collect_metadata()` — `os.uname()` for kernel + arch, `/etc/os-release` for distro. Called once at startup and cached.
+- `collect_docker_logs(socket, containers, state, exclude)` — (m79) per running
+  container, fetches new log lines over the Docker socket with an incremental
+  `since` cursor kept per container in `state` (a container's first interval
+  seeds from a short tail). Demuxes the multiplexed stream, parses each line's
+  RFC3339 timestamp, caps the whole batch at a byte limit (a marker line records
+  a truncation; the deferred lines come next interval). Folded into the sample as
+  `docker_logs`; omitted when empty.
 - `build_payload()` — assembles a snapshot from all collectors into one dict.
 - `post_payload(url, token, payloads)` — POSTs a list of samples, returns success/failure.
 - `RingBuffer(maxlen=20)` — tiny FIFO wrapper, drops oldest when full.
-- `main_loop()` — the 30s loop: collect → try POST → on failure push to buffer + backoff → on success flush buffer.
+- `main_loop()` — the 30s loop: collect → try POST → on failure push to buffer + backoff → on success flush buffer. Container logs are stripped from a sample before it's buffered (metrics survive an outage; stale logs are dropped).
 
 **Target: ~300 lines total including docstrings.** More than that is a smell that the agent is over-scoping.
 
@@ -255,6 +269,7 @@ Content-Type: application/json
 - **`metadata` is sent on every POST**, not just on change. Server-side diff detects actual changes and only writes on change. Cost per POST is one dict — negligible. Benefit: server can cleanly detect agent restarts.
 - **Raw bytes, not percentages, for memory and storage.** Percentages are derived server-side. Changing the "what counts as used" math doesn't require re-releasing the agent.
 - **CPU is the one exception** — reported as a percentage because it's inherently a derivative (delta over time), not a snapshot. The agent must sample twice to compute it.
+- **Container logs (m79) ride in the same push** as `docker_logs`: a list of `{container, stream, ts, line}` records — incremental since the previous interval. This pushes the *same direction* as metrics, so batched log history needs no inbound channel (only sub-second live-follow would). The server ingests them into `docker_logs`, enforces the global toggle + exclude list on ingest, and bounds storage with a per-container size+age ring. `docker_logs` is omitted when there's nothing new.
 
 ### Server expansion into `PluginMetric` rows
 

docs/reference/playbook-authoring.md

@@ -0,0 +1,191 @@
+# Authoring Steward-friendly Ansible playbooks
+
+This is the contract between a playbook and Steward's run UI. Follow it and a
+playbook drops into Steward with a description, fill-in variable fields, correct
+targeting, and credentials supplied automatically — no per-playbook wiring.
+
+It applies to **any** playbook in a configured source (bundled, the writable
+local source, or a git source), including third-party ones.
+
+---
+
+## 1. Describe what it does — `# description:`
+
+Steward shows a one-line description when a playbook is selected in the run form.
+
+```yaml
+---
+# description: Reclaim disk on Docker/Swarm nodes by pruning unused images and build cache.
+- name: Docker system prune
+  hosts: all
+  ...
+```
+
+- Format: a comment line `# description: <text>` anywhere in the file. **First
+  match wins.** Case-insensitive on the `description:` key.
+- Why a comment and not a key: Ansible rejects unknown *play* keys (you can't add
+  `description:` to a play), so a comment is the portable place. It survives
+  `ansible-playbook` untouched.
+- Fallback: if there's no `# description:` comment, Steward uses the **first
+  play's `name:`**. So always give your play a meaningful `name:` even without
+  the comment.
+- Keep it to one readable line. Longer "how to use" notes can go in additional
+  normal comments — Steward only reads the `description:` line.
+
+## 2. Declare tunables in `vars:` — they become fill-in fields
+
+Every **scalar** entry in a play's `vars:` block becomes an editable field in the
+run form, with the default shown as the input's placeholder.
+
+```yaml
+  vars:
+    prune_all_images: false      # → checkbox-ish text field, placeholder "default: false"
+    keep_last_days: 7            # → field, placeholder "default: 7"
+    registry_url: ""             # → field, placeholder "no default"
+```
+
+- **Blank field = use the default.** Steward only sends fields the operator
+  actually fills, so an untouched field falls through to the playbook/inventory
+  default rather than overriding it.
+- Only scalars (string/int/float/bool) surface as fields. Lists/dicts are
+  skipped — set those in the playbook or via inventory group/host vars.
+- Values are delivered as **extra-vars** (`-e`), which are the **highest**
+  precedence in Ansible — they override the `vars:` defaults. (This is why the
+  default can be empty and still be safely overridden at run time.)
+
+### `vars_prompt:` also works
+
+Steward reads `vars_prompt` too. Use it when you want an explicit prompt or a
+required value:
+
+```yaml
+  vars_prompt:
+    - name: release_tag
+      prompt: "Which release to deploy?"     # shown as the field label
+      # no default → Steward marks the field REQUIRED
+    - name: admin_password
+      prompt: "Admin password"
+      private: true                          # → masked field, never stored
+```
+
+## 3. Secrets — name them so they're masked and not persisted
+
+A field is treated as **secret** (rendered masked, and its value is **never
+written to the DB / run history**) when either:
+
+- the variable name contains `password`, `passwd`, `secret`, `token`,
+  `api_key` / `apikey`, `private_key`, or `credential` (case-insensitive), **or**
+- it's a `vars_prompt` entry with `private: true` (Ansible's default for
+  vars_prompt is private).
+
+So name sensitive variables accordingly (`db_password`, `api_token`,
+`vault_secret`) and they're handled safely with no extra config. Non-secret
+run-time vars are persisted (so scheduled runs can reuse them); secret ones are
+passed to the run only.
+
+## 4. Target with `hosts: all`
+
+Steward builds the inventory itself from the **target / group** the operator
+picks in the run form (or the single host on a host page). Your play should:
+
+```yaml
+  hosts: all          # run against whatever Steward scoped to
+```
+
+- Don't hardcode hostnames or rely on a checked-in inventory for Steward runs
+  (Steward generates a fresh inventory per run).
+- Per-host connection vars (`ansible_host`, plus anything you set on the target
+  in **Ansible → Inventory**) arrive as inventory host vars.
+- The run form's **Limit** / **Tags** map to `--limit` / `--tags`.
+
+## 5. Privileges & connection — don't put credentials in the playbook
+
+Steward supplies SSH and become for you:
+
+- Steady-state runs connect as the managed **`steward`** account using Steward's
+  managed key; that account has **passwordless sudo**. So just use
+  `become: true` where you need root.
+- First-contact provisioning uses a one-time bootstrap user/password the
+  operator enters (never stored).
+- Never embed SSH keys, passwords, or `ansible_user`/`ansible_ssh_pass` in the
+  playbook. Connection identity is global (Settings → Ansible) or per-target.
+
+## 6. Be idempotent
+
+Steward re-runs playbooks (updates, schedules, retries). Use modules that
+converge (state-based) rather than ad-hoc `command:`/`shell:` where possible, so
+re-runs are safe.
+
+---
+
+## What Steward reads from a playbook (summary)
+
+| Source in the playbook | What Steward does with it |
+|---|---|
+| `# description: <text>` comment | Description shown on selection (first match) |
+| first play `name:` | Description fallback |
+| `# steward:category: <text>` | Grouping badge in the browse list + run form |
+| `# steward:confirm: true` | Requires a confirmation tick before the run launches |
+| `vars:` scalar entries | Run-time fill-in fields (placeholder = default) |
+| `vars_prompt:` entries | Run-time fields (required if no default) |
+| secret-looking var name / `private: true` | Field masked + value not persisted |
+| `hosts:` | Expected to be `all`; Steward provides the inventory |
+
+Everything else (SSH user/key, become password, the inventory, `steward_token`
+etc. for the agent playbooks) is injected by Steward at run time.
+
+## Metadata comments
+
+Steward reads metadata from magic comments (Ansible rejects unknown play keys,
+so comments are the portable place). Two forms:
+
+- `# description: <text>` — the description (see §1).
+- `# steward:<key>: <value>` — the namespaced metadata block. First match per
+  key wins; keys are case-insensitive.
+
+**Implemented `# steward:` keys:**
+
+| Key | Example | Effect |
+|---|---|---|
+| `category` | `# steward:category: maintenance` | Free-text grouping label. Shown as a badge in the browse list and on the run form. Purely organizational. |
+| `confirm` | `# steward:confirm: true` | Marks the playbook as significant/destructive. The run form then requires an explicit confirmation tick before it can be launched. Use for data-loss-capable plays (prune with volumes, resets, etc.). Truthy values: `true`/`yes`/`1`/`on`. |
+| `description` | `# steward:description: ...` | Alias for `# description:` (the unprefixed form takes precedence if both exist). |
+
+```yaml
+---
+# description: Reclaim disk on Docker/Swarm nodes by pruning unused data.
+# steward:category: maintenance
+# steward:confirm: true
+- name: Docker system prune
+  hosts: all
+  ...
+```
+
+Other `# steward:<key>:` keys are simply ignored today — the namespace is
+reserved, so it's safe to add ones Steward doesn't yet understand without
+breaking anything, but only `category`, `confirm`, and `description` do something.
+
+## Minimal annotated example
+
+```yaml
+---
+# description: Restart a systemd service and confirm it came back up.
+- name: Restart a service
+  hosts: all
+  become: true
+  gather_facts: false
+  vars:
+    service_name: ""        # required-ish: operator fills it in the run form
+  tasks:
+    - name: Validate input
+      ansible.builtin.assert:
+        that: service_name | default('') | length > 0
+        fail_msg: "Set service_name."
+    - name: Restart
+      ansible.builtin.systemd:
+        name: "{{ service_name }}"
+        state: restarted
+    - name: Confirm active
+      ansible.builtin.command: "systemctl is-active {{ service_name }}"
+      changed_when: false
+```

plugins/docker/__init__.py

@@ -13,12 +13,42 @@ def setup(app: "Quart") -> None:
     _app = app
     from .models import DockerContainer, DockerMetric  # noqa: F401 — register with Base
 
+    # Publish the per-host persist hook so the host agent can hand off the
+    # `docker` array from each sample without importing our models (opportunistic
+    # synergy — host_agent gates on has_capability and no-ops if we're disabled).
+    # required_role=viewer: this is a trusted server-side data-plane write driven
+    # by the authenticated agent ingest, not a user-facing privileged action.
+    from steward.core.capabilities import register_capability
+    from steward.models.users import UserRole
+    from .ingest import persist_host_docker
+    from .retention import run_docker_retention
+    register_capability(
+        "docker.persist_host_samples", persist_host_docker,
+        label="Persist host Docker samples",
+        description="Store per-host container state + metrics pushed by the host agent.",
+        required_role=UserRole.viewer,
+    )
+    # Roll up + prune Docker time-series, driven by the core cleanup task without
+    # it importing our models. Same trusted server-side data-plane role as above.
+    register_capability(
+        "docker.run_retention", run_docker_retention,
+        label="Run Docker retention",
+        description="Roll up old docker_metrics to hourly + prune stale metrics/events.",
+        required_role=UserRole.viewer,
+    )
+
 
 def get_scheduled_tasks() -> list:
-    from .scheduler import make_task
-    return [make_task(_app)]
+    # Collection is now agent-driven (pushed to the host_agent ingest); the
+    # central socket scrape was removed. No periodic task to register.
+    return []
 
 
 def get_blueprint():
     from .routes import docker_bp
     return docker_bp
+
+
+def get_nav() -> list[dict]:
+    # Surfaces in the sidebar's Infrastructure group (fleet-wide container view).
+    return [{"label": "Docker", "href": "/plugins/docker/"}]

plugins/docker/dedup.py

@@ -0,0 +1,27 @@
+"""Model-free Docker view helpers.
+
+Kept separate from routes.py/models.py so tests can import it directly: importing
+a module that re-imports plugins.docker.models AFTER the app has loaded the docker
+plugin raises "Table 'docker_containers' is already defined". This module touches
+no ORM models, so it's safe to import in either environment.
+"""
+from __future__ import annotations
+
+
+def dedup_by_container_id(containers: list) -> list:
+    """Collapse containers double-reported across hosts. Swarm-aware agents on
+    every manager list cluster-wide tasks, so the same container (identical
+    container_id) arrives once per manager and would otherwise be counted N times.
+    Keep the first occurrence per non-empty container_id — callers order
+    running-first, so a running report wins over a stale stopped one. Rows without
+    a container_id (older agents) can't be matched, so they're kept as-is."""
+    seen: set[str] = set()
+    out = []
+    for c in containers:
+        cid = (c.container_id or "").strip()
+        if cid:
+            if cid in seen:
+                continue
+            seen.add(cid)
+        out.append(c)
+    return out

plugins/docker/ingest.py

@@ -0,0 +1,390 @@
+# plugins/docker/ingest.py
+"""Persist host-scoped Docker samples pushed by the host agent.
+
+Published as the "docker.persist_host_samples" capability (see __init__.setup),
+so the host_agent plugin can hand off a sample's `docker` array (and, on a swarm
+manager, its `swarm` object) WITHOUT importing the docker models — the coupling
+is opportunistic and degrades to a no-op when the docker plugin is disabled.
+Runs inside the caller's open transaction (the ingest handler's SAVEPOINT);
+never opens or commits its own.
+"""
+from __future__ import annotations
+
+import json
+from datetime import datetime
+
+from sqlalchemy import delete, select
+
+
+# Stopped/terminal container states (anything not "running"/"paused"/"restarting").
+_STOPPED_STATES = {"exited", "dead", "stopped"}
+
+
+def _parse_started_at(value) -> datetime | None:
+    """Parse the agent's ISO-8601 started_at string back to a datetime."""
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        return datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+
+
+def _derive_events(old_state: dict, new_containers: list) -> list:
+    """Diff a fresh snapshot against stored per-container state → lifecycle events.
+
+    Pure function (no DB) so it's unit-testable. `old_state` maps name → the
+    previously stored {status, health, oom_killed, exit_code}; `new_containers`
+    is the newest snapshot's list of container dicts. Returns a list of
+    (container_name, event, detail) tuples:
+
+      start         — a container transitions into running (or a genuinely new
+                      container appears already running)
+      stop          — running → not-running, or a running container is removed
+      die           — same as stop but with a non-zero exit code (abnormal)
+      oom           — OOMKilled flips False→True
+      health_change — HEALTHCHECK status string changes
+
+    The CALLER skips this entirely on a host's first-ever snapshot (empty
+    old_state) so the baseline doesn't emit a spurious "start" per existing
+    container — a later-appearing container still gets one because by then
+    old_state is populated and that container's prior entry is simply absent.
+    """
+    events: list = []
+    new_by_name = {c["name"]: c for c in new_containers if c.get("name")}
+
+    for name, c in new_by_name.items():
+        new_status = (c.get("status") or "").lower()
+        new_running = new_status == "running"
+        new_oom = bool(c.get("oom_killed"))
+        new_health = c.get("health")
+        new_exit = c.get("exit_code")
+        old = old_state.get(name)
+
+        if old is None:
+            # Newly observed container — only a start is meaningful (we have no
+            # prior state to diff a death against).
+            if new_running:
+                events.append((name, "start", c.get("image") or None))
+            continue
+
+        old_running = (old.get("status") or "").lower() == "running"
+        if new_running and not old_running:
+            events.append((name, "start", c.get("image") or None))
+        elif old_running and not new_running:
+            if new_exit not in (None, 0):
+                events.append((name, "die", f"exit {new_exit}"))
+            else:
+                events.append((name, "stop", None))
+
+        if new_oom and not bool(old.get("oom_killed")):
+            events.append((name, "oom", None))
+
+        if new_health != old.get("health") and (new_health or old.get("health")):
+            events.append((name, "health_change",
+                           f"{old.get('health') or '?'}→{new_health or '?'}"))
+
+    # A running container that vanished from the listing entirely (removed).
+    for name, old in old_state.items():
+        if name not in new_by_name and (old.get("status") or "").lower() == "running":
+            events.append((name, "stop", "removed"))
+
+    return events
+
+
+def _log_rows(log_batches, host_id: str):
+    """Pure: flatten the agent's log batches → docker_logs row kwargs (no DB).
+
+    `log_batches` is a list of (recorded_at, records) where each record is
+    {"container", "stream", "ts", "line"}. A line's own Docker timestamp wins;
+    recorded_at is the fallback when the agent couldn't parse one. Drops the
+    agent's advisory truncation marker (container "_steward" — it signals a
+    deferral, not a real container), malformed records, and lineless records;
+    normalises an unknown stream to stdout. Unit-testable in isolation.
+    """
+    for recorded_at, records in log_batches:
+        if not isinstance(records, list):
+            continue
+        for rec in records:
+            if not isinstance(rec, dict):
+                continue
+            name = rec.get("container")
+            line = rec.get("line")
+            if not name or name == "_steward" or line is None:
+                continue
+            ts = _parse_started_at(rec.get("ts")) or recorded_at
+            stream = rec.get("stream")
+            if stream not in ("stdout", "stderr"):
+                stream = "stdout"
+            yield {"host_id": host_id, "container_name": str(name)[:255],
+                   "ts": ts, "stream": stream, "line": str(line)}
+
+
+async def _persist_logs(session, host, log_batches) -> None:
+    """Append pushed container log lines (one row per line) for this host.
+
+    Time-series / append-only — the per-container size+age ring (retention)
+    bounds growth, so a chatty container just keeps a shorter window. The global
+    toggle + exclude list (Settings, no restart) are enforced here: this is the
+    authoritative drop point, since the push model has no channel to tell an
+    agent to stop collecting.
+    """
+    from steward.core.settings import get_setting
+    from .models import DockerLog
+
+    if not await get_setting(session, "docker.logs.enabled"):
+        return
+    exclude = set(await get_setting(session, "docker.logs.exclude") or ())
+    for row in _log_rows(log_batches, host.id):
+        if row["container_name"] in exclude:
+            continue
+        session.add(DockerLog(**row))
+
+
+async def _persist_swarm(session, host, swarm: dict) -> None:
+    """Upsert this manager's swarm topology; drop rows no longer reported.
+
+    Current-state tables (not time-series): a manager re-reports its full
+    services/nodes set every sample, so we upsert what's present and delete what
+    isn't (scoped to this host so two managers don't clobber each other).
+    """
+    from datetime import timezone
+    from .models import DockerSwarmNode, DockerSwarmService
+
+    now = datetime.now(timezone.utc)
+
+    services = swarm.get("services") or []
+    seen_services: set[str] = set()
+    for s in services:
+        name = s.get("service_name")
+        if not name:
+            continue
+        seen_services.add(name)
+        row = await session.get(DockerSwarmService, (host.id, name))
+        if row is None:
+            row = DockerSwarmService(host_id=host.id, service_name=name)
+            session.add(row)
+        row.mode = s.get("mode") or "replicated"
+        row.desired = int(s.get("desired") or 0)
+        row.running = int(s.get("running") or 0)
+        row.image = s.get("image")
+        row.placement_json = json.dumps(s.get("placement") or [])
+        row.updated_at = now
+    stale_services = delete(DockerSwarmService).where(
+        DockerSwarmService.host_id == host.id)
+    if seen_services:
+        stale_services = stale_services.where(
+            DockerSwarmService.service_name.notin_(seen_services))
+    await session.execute(stale_services)
+
+    nodes = swarm.get("nodes") or []
+    seen_nodes: set[str] = set()
+    for n in nodes:
+        nid = n.get("node_id")
+        if not nid:
+            continue
+        seen_nodes.add(nid)
+        row = await session.get(DockerSwarmNode, (host.id, nid))
+        if row is None:
+            row = DockerSwarmNode(host_id=host.id, node_id=nid)
+            session.add(row)
+        row.hostname = n.get("hostname") or ""
+        row.role = n.get("role") or "worker"
+        row.availability = n.get("availability") or "active"
+        row.status = n.get("status") or "unknown"
+        row.leader = bool(n.get("leader", False))
+        row.updated_at = now
+    stale_nodes = delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id)
+    if seen_nodes:
+        stale_nodes = stale_nodes.where(DockerSwarmNode.node_id.notin_(seen_nodes))
+    await session.execute(stale_nodes)
+
+
+async def _persist_disk(session, host, disk: dict) -> None:
+    """Upsert this host's /system/df summary + replace its image storage rows.
+
+    Current-state (not time-series): the agent re-reports the full summary each
+    interval, so we overwrite the single summary row and replace the image set
+    (delete rows no longer present, upsert the rest), scoped to this host.
+    """
+    from datetime import timezone
+    from .models import DockerDiskUsage, DockerImage
+
+    now = datetime.now(timezone.utc)
+
+    summary = await session.get(DockerDiskUsage, host.id)
+    if summary is None:
+        summary = DockerDiskUsage(host_id=host.id)
+        session.add(summary)
+    summary.layers_size = int(disk.get("layers_size") or 0)
+    summary.images_size = int(disk.get("images_size") or 0)
+    summary.images_reclaimable = int(disk.get("images_reclaimable") or 0)
+    summary.containers_size = int(disk.get("containers_size") or 0)
+    summary.volumes_size = int(disk.get("volumes_size") or 0)
+    summary.build_cache_size = int(disk.get("build_cache_size") or 0)
+    summary.images_total = int(disk.get("images_total") or 0)
+    summary.images_active = int(disk.get("images_active") or 0)
+    summary.containers_count = int(disk.get("containers_count") or 0)
+    summary.volumes_count = int(disk.get("volumes_count") or 0)
+    summary.updated_at = now
+
+    images = disk.get("images") or []
+    seen: set[str] = set()
+    for im in images:
+        iid = im.get("image_id")
+        if not iid or iid in seen:
+            continue
+        seen.add(iid)
+        row = await session.get(DockerImage, (host.id, iid))
+        if row is None:
+            row = DockerImage(host_id=host.id, image_id=iid)
+            session.add(row)
+        row.repo_tag = (im.get("repo_tag") or "<none>")[:512]
+        row.size = int(im.get("size") or 0)
+        row.shared_size = int(im.get("shared_size") or 0)
+        row.containers = int(im.get("containers") or 0)
+        row.updated_at = now
+    stale = delete(DockerImage).where(DockerImage.host_id == host.id)
+    if seen:
+        stale = stale.where(DockerImage.image_id.notin_(seen))
+    await session.execute(stale)
+
+
+async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
+                              logs=None) -> None:
+    """Upsert containers + time-series + lifecycle events + swarm for one host.
+
+    `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
+    one entry per ingested sample carrying docker data (usually one; more when
+    the agent flushes a backlog). Every snapshot contributes time-series
+    DockerMetric rows; the newest snapshot drives current container state, the
+    alert pipeline, and lifecycle-event derivation. `swarm` is the newest
+    sample's swarm object (or None off managers) — persisted when present.
+    `disk` is the newest sample's /system/df summary (or None on Docker-less
+    hosts) — persisted when present. `logs` is a list of (recorded_at, records)
+    log batches (m79) — appended one row per line when present, independent of
+    whether this sample also carried container metrics.
+    """
+    from steward.core.alerts import record_metric
+    from .models import DockerContainer, DockerEvent, DockerMetric
+
+    if swarm is not None:
+        await _persist_swarm(session, host, swarm)
+    if disk is not None:
+        await _persist_disk(session, host, disk)
+    if logs:
+        await _persist_logs(session, host, logs)
+
+    if not snapshots:
+        return
+    ordered = sorted(snapshots, key=lambda s: s[0])
+    latest_at, latest_containers = ordered[-1]
+
+    # Time-series points for every snapshot (running containers with a CPU read).
+    for recorded_at, containers in ordered:
+        for c in containers:
+            if c.get("status") == "running" and c.get("cpu_pct") is not None:
+                session.add(DockerMetric(
+                    host_id=host.id,
+                    container_name=c["name"],
+                    scraped_at=recorded_at,
+                    cpu_pct=c["cpu_pct"],
+                    mem_pct=c.get("mem_pct") or 0.0,
+                    mem_usage_bytes=c.get("mem_usage_bytes") or 0,
+                ))
+
+    # Snapshot of stored per-container state BEFORE the upsert overwrites it —
+    # used to diff lifecycle events. Skip event derivation on the host's first
+    # snapshot (empty state) so we don't emit a start per pre-existing container.
+    old_rows = (await session.execute(
+        select(DockerContainer).where(DockerContainer.host_id == host.id)
+    )).scalars().all()
+    old_state = {
+        r.name: {"status": r.status, "health": r.health,
+                 "oom_killed": r.oom_killed, "exit_code": r.exit_code}
+        for r in old_rows
+    }
+    if old_state:
+        for name, event, detail in _derive_events(old_state, latest_containers):
+            session.add(DockerEvent(
+                host_id=host.id, container_name=name,
+                event=event, detail=detail, at=latest_at,
+            ))
+
+    # Current state + alerts from the newest snapshot only.
+    for c in latest_containers:
+        name = c.get("name")
+        if not name:
+            continue
+        existing = await session.get(DockerContainer, (host.id, name))
+        if existing is None:
+            existing = DockerContainer(host_id=host.id, name=name)
+            session.add(existing)
+        existing.container_id = c.get("container_id", "") or ""
+        existing.image = c.get("image", "") or ""
+        existing.status = c.get("status", "unknown") or "unknown"
+        existing.cpu_pct = c.get("cpu_pct")
+        existing.mem_usage_bytes = c.get("mem_usage_bytes")
+        existing.mem_limit_bytes = c.get("mem_limit_bytes")
+        existing.mem_pct = c.get("mem_pct")
+        existing.ports_json = json.dumps(c.get("ports") or [])
+        existing.started_at = _parse_started_at(c.get("started_at"))
+        existing.scraped_at = latest_at
+        # Enrichment (agent ≥ 1.4.0; .get keeps older agents working — fields
+        # stay None/0 when absent from the payload).
+        existing.restart_count = c.get("restart_count", 0) or 0
+        existing.health = c.get("health")
+        existing.exit_code = c.get("exit_code")
+        existing.oom_killed = bool(c.get("oom_killed", False))
+        existing.compose_project = c.get("compose_project")
+        existing.service_name = c.get("service_name")
+        existing.task_id = c.get("task_id")
+        existing.node_id = c.get("node_id")
+        existing.net_rx_bytes = c.get("net_rx_bytes")
+        existing.net_tx_bytes = c.get("net_tx_bytes")
+        existing.blk_read_bytes = c.get("blk_read_bytes")
+        existing.blk_write_bytes = c.get("blk_write_bytes")
+
+        # Alert pipeline — resource is host-scoped so containers of the same name
+        # on different hosts don't collide in the metric/alert namespace.
+        resource = f"{host.name}/{name}"
+        if c.get("status") == "running" and c.get("cpu_pct") is not None:
+            await record_metric(
+                session=session, source_module="docker",
+                resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
+            )
+            if c.get("mem_pct") is not None:
+                await record_metric(
+                    session=session, source_module="docker",
+                    resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
+                )
+        # Restart count is alertable regardless of state (crash-looping matters
+        # most while the container is down/restarting).
+        await record_metric(
+            session=session, source_module="docker",
+            resource_name=resource, metric_name="restart_count",
+            value=float(c.get("restart_count", 0) or 0),
+        )
+        # Health → 1.0/0.0 only for containers that actually define a HEALTHCHECK
+        # (health is None otherwise — recording 0 would false-alarm every plain
+        # container).
+        health = c.get("health")
+        if health in ("healthy", "unhealthy", "starting"):
+            await record_metric(
+                session=session, source_module="docker",
+                resource_name=resource, metric_name="is_healthy",
+                value=1.0 if health == "healthy" else 0.0,
+            )
+
+    # Reap containers that vanished from the host's latest listing. Without this,
+    # removed one-shot containers (CI job runners, buildkit builders, codex jobs)
+    # accumulate forever as permanent "stopped" rows and inflate every count.
+    # The listing is authoritative (event derivation already treats absence as
+    # removal), so anything not in the newest snapshot no longer exists on the
+    # host. Mirrors the image/swarm/disk persisters. An empty snapshot legitimately
+    # means the host has no containers, so the unfiltered delete is correct.
+    seen_names = {c["name"] for c in latest_containers if c.get("name")}
+    reap = delete(DockerContainer).where(DockerContainer.host_id == host.id)
+    if seen_names:
+        reap = reap.where(DockerContainer.name.notin_(seen_names))
+    await session.execute(reap)

plugins/docker/migrations/versions/docker_002_host_scoped.py

@@ -0,0 +1,94 @@
+"""Docker collection goes per-host: add host_id, re-key by (host_id, name)
+
+Collection moved from the central single-socket scrape to the host agent, so
+containers are now scoped to the host that reported them. docker_containers is
+re-keyed (host_id, name) and docker_metrics gains host_id.
+
+Dev-only posture (rule 122): the old tables only ever held the Steward box's
+own containers (a single global namespace), which are disposable — so this
+DROP+recreates rather than backfilling a host_id onto orphan rows.
+
+Revision ID: docker_002_host_scoped
+Revises: docker_001_initial
+Create Date: 2026-06-18
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_002_host_scoped"
+down_revision: Union[str, None] = "docker_001_initial"
+branch_labels: Union[str, Sequence[str], None] = None
+# FK targets hosts.id (created in 0002_core_monitors) — make the edge explicit.
+depends_on: Union[str, Sequence[str], None] = ("0002_core_monitors",)
+
+
+def upgrade() -> None:
+    op.drop_table("docker_metrics")
+    op.drop_table("docker_containers")
+
+    op.create_table(
+        "docker_containers",
+        sa.Column("host_id", sa.String(36),
+                  sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True),
+        sa.Column("name", sa.String(255), primary_key=True),
+        sa.Column("container_id", sa.String(64), nullable=False, server_default=""),
+        sa.Column("image", sa.String(512), nullable=False, server_default=""),
+        sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
+        sa.Column("cpu_pct", sa.Float, nullable=True),
+        sa.Column("mem_usage_bytes", sa.Integer, nullable=True),
+        sa.Column("mem_limit_bytes", sa.Integer, nullable=True),
+        sa.Column("mem_pct", sa.Float, nullable=True),
+        sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+    )
+
+    op.create_table(
+        "docker_metrics",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("host_id", sa.String(36),
+                  sa.ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("container_name", sa.String(255), nullable=False),
+        sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"),
+        sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"),
+        sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"),
+    )
+    op.create_index("ix_docker_metrics_host_id", "docker_metrics", ["host_id"])
+    op.create_index("ix_docker_metrics_container_name", "docker_metrics",
+                    ["container_name"])
+    op.create_index("ix_docker_metrics_scraped_at", "docker_metrics", ["scraped_at"])
+    op.create_index("ix_docker_metrics_host_container_time", "docker_metrics",
+                    ["host_id", "container_name", "scraped_at"])
+
+
+def downgrade() -> None:
+    op.drop_table("docker_metrics")
+    op.drop_table("docker_containers")
+
+    op.create_table(
+        "docker_containers",
+        sa.Column("name", sa.String(255), primary_key=True),
+        sa.Column("container_id", sa.String(64), nullable=False, server_default=""),
+        sa.Column("image", sa.String(512), nullable=False, server_default=""),
+        sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
+        sa.Column("cpu_pct", sa.Float, nullable=True),
+        sa.Column("mem_usage_bytes", sa.Integer, nullable=True),
+        sa.Column("mem_limit_bytes", sa.Integer, nullable=True),
+        sa.Column("mem_pct", sa.Float, nullable=True),
+        sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
+    )
+    op.create_table(
+        "docker_metrics",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("container_name", sa.String(255), nullable=False, index=True),
+        sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True),
+        sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"),
+        sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"),
+        sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"),
+    )

plugins/docker/migrations/versions/docker_003_container_enrichment.py

@@ -0,0 +1,41 @@
+"""Docker container enrichment: health, exit/restart, grouping, I/O counters
+
+Adds the fields the agent (≥1.4.0) now reports per container beyond the basic
+state: health status, exit code, OOM flag, compose/swarm grouping labels, and
+cumulative network/block I/O counters. Additive columns — no data loss, so no
+DROP+recreate needed here.
+
+Revision ID: docker_003_container_enrichment
+Revises: docker_002_host_scoped
+Create Date: 2026-06-19
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_003_container_enrichment"
+down_revision: Union[str, None] = "docker_002_host_scoped"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("docker_containers", sa.Column("health", sa.String(16), nullable=True))
+    op.add_column("docker_containers", sa.Column("exit_code", sa.Integer, nullable=True))
+    op.add_column("docker_containers",
+                  sa.Column("oom_killed", sa.Boolean, nullable=False, server_default=sa.false()))
+    op.add_column("docker_containers", sa.Column("compose_project", sa.String(255), nullable=True))
+    op.add_column("docker_containers", sa.Column("service_name", sa.String(255), nullable=True))
+    op.add_column("docker_containers", sa.Column("task_id", sa.String(64), nullable=True))
+    op.add_column("docker_containers", sa.Column("node_id", sa.String(64), nullable=True))
+    op.add_column("docker_containers", sa.Column("net_rx_bytes", sa.BigInteger, nullable=True))
+    op.add_column("docker_containers", sa.Column("net_tx_bytes", sa.BigInteger, nullable=True))
+    op.add_column("docker_containers", sa.Column("blk_read_bytes", sa.BigInteger, nullable=True))
+    op.add_column("docker_containers", sa.Column("blk_write_bytes", sa.BigInteger, nullable=True))
+
+
+def downgrade() -> None:
+    for col in ("blk_write_bytes", "blk_read_bytes", "net_tx_bytes", "net_rx_bytes",
+                "node_id", "task_id", "service_name", "compose_project",
+                "oom_killed", "exit_code", "health"):
+        op.drop_column("docker_containers", col)

plugins/docker/migrations/versions/docker_004_events_swarm.py

@@ -0,0 +1,77 @@
+"""Docker lifecycle events + swarm topology tables
+
+Adds the storage for milestone-77 monitoring depth that doesn't fit on the
+per-container row:
+
+  * docker_events — lifecycle (start/stop/die/oom/health_change) derived by
+    diffing consecutive host snapshots; host-scoped, indexed for both timeline
+    lookups (host_id, container_name, at) and retention pruning (at).
+  * docker_swarm_services / docker_swarm_nodes — manager-reported Swarm
+    topology (desired-vs-running replicas, node role/availability/status).
+
+All new tables — purely additive, so no DROP+recreate. `event`/`mode`/`role`
+etc. are plain strings (no CHECK whitelist), matching how docker_containers
+models `status`; keeps the value sets open without a migration per addition
+(so rule 36 doesn't apply here).
+
+Revision ID: docker_004_events_swarm
+Revises: docker_003_container_enrichment
+Create Date: 2026-06-19
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_004_events_swarm"
+down_revision: Union[str, None] = "docker_003_container_enrichment"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "docker_events",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("host_id", sa.String(36),
+                  sa.ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("container_name", sa.String(255), nullable=False),
+        sa.Column("event", sa.String(16), nullable=False),
+        sa.Column("detail", sa.String(255), nullable=True),
+        sa.Column("at", sa.DateTime(timezone=True), nullable=False),
+    )
+    op.create_index("ix_docker_events_host_container_time", "docker_events",
+                    ["host_id", "container_name", "at"])
+    op.create_index("ix_docker_events_at", "docker_events", ["at"])
+
+    op.create_table(
+        "docker_swarm_services",
+        sa.Column("host_id", sa.String(36),
+                  sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True),
+        sa.Column("service_name", sa.String(255), primary_key=True),
+        sa.Column("mode", sa.String(16), nullable=False, server_default="replicated"),
+        sa.Column("desired", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("running", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("image", sa.String(512), nullable=True),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+    )
+
+    op.create_table(
+        "docker_swarm_nodes",
+        sa.Column("host_id", sa.String(36),
+                  sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True),
+        sa.Column("node_id", sa.String(64), primary_key=True),
+        sa.Column("hostname", sa.String(255), nullable=False, server_default=""),
+        sa.Column("role", sa.String(16), nullable=False, server_default="worker"),
+        sa.Column("availability", sa.String(16), nullable=False, server_default="active"),
+        sa.Column("status", sa.String(16), nullable=False, server_default="unknown"),
+        sa.Column("leader", sa.Boolean, nullable=False, server_default=sa.false()),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("docker_swarm_nodes")
+    op.drop_table("docker_swarm_services")
+    op.drop_index("ix_docker_events_at", table_name="docker_events")
+    op.drop_index("ix_docker_events_host_container_time", table_name="docker_events")
+    op.drop_table("docker_events")

plugins/docker/migrations/versions/docker_005_swarm_placement.py

@@ -0,0 +1,29 @@
+"""Docker swarm service placement column
+
+Adds docker_swarm_services.placement_json — the task→node placement of a
+service's running replicas, captured from the agent's swarm payload (a manager
+sees every task, so this records cross-node placement the local container rows
+can't). Additive column; no DROP+recreate.
+
+Revision ID: docker_005_swarm_placement
+Revises: docker_004_events_swarm
+Create Date: 2026-06-19
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_005_swarm_placement"
+down_revision: Union[str, None] = "docker_004_events_swarm"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("docker_swarm_services",
+                  sa.Column("placement_json", sa.Text, nullable=False,
+                            server_default="[]"))
+
+
+def downgrade() -> None:
+    op.drop_column("docker_swarm_services", "placement_json")

plugins/docker/migrations/versions/docker_006_metric_rollup.py

@@ -0,0 +1,45 @@
+"""Docker hourly metric rollup table
+
+Adds docker_metrics_hourly — the coarse series that retention rolls raw
+docker_metrics into before pruning them, so multi-day history stays cheap.
+One row per (host, container, hour bucket); the unique constraint is the
+conflict target for the idempotent rollup upsert. Additive create_table.
+
+Revision ID: docker_006_metric_rollup
+Revises: docker_005_swarm_placement
+Create Date: 2026-06-19
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_006_metric_rollup"
+down_revision: Union[str, None] = "docker_005_swarm_placement"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "docker_metrics_hourly",
+        sa.Column("id", sa.String(length=36), nullable=False),
+        sa.Column("host_id", sa.String(length=36), nullable=False),
+        sa.Column("container_name", sa.String(length=255), nullable=False),
+        sa.Column("bucket", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("cpu_pct", sa.Float(), nullable=False, server_default="0"),
+        sa.Column("mem_pct", sa.Float(), nullable=False, server_default="0"),
+        sa.Column("mem_usage_bytes", sa.BigInteger(), nullable=False, server_default="0"),
+        sa.Column("sample_count", sa.Integer(), nullable=False, server_default="0"),
+        sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("host_id", "container_name", "bucket",
+                            name="uq_docker_metrics_hourly_bucket"),
+    )
+    op.create_index("ix_docker_metrics_hourly_bucket",
+                    "docker_metrics_hourly", ["bucket"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_docker_metrics_hourly_bucket",
+                  table_name="docker_metrics_hourly")
+    op.drop_table("docker_metrics_hourly")

plugins/docker/migrations/versions/docker_007_disk_usage.py

@@ -0,0 +1,55 @@
+"""Docker disk usage + image storage tables
+
+Adds docker_disk_usage (one per-host /system/df summary row) and docker_images
+(the heavy-hitter image storage records). Both current-state, host-scoped,
+re-reported each interval. Additive create_table.
+
+Revision ID: docker_007_disk_usage
+Revises: docker_006_metric_rollup
+Create Date: 2026-06-19
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_007_disk_usage"
+down_revision: Union[str, None] = "docker_006_metric_rollup"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "docker_disk_usage",
+        sa.Column("host_id", sa.String(length=36), nullable=False),
+        sa.Column("layers_size", sa.BigInteger(), nullable=False, server_default="0"),
+        sa.Column("images_size", sa.BigInteger(), nullable=False, server_default="0"),
+        sa.Column("images_reclaimable", sa.BigInteger(), nullable=False, server_default="0"),
+        sa.Column("containers_size", sa.BigInteger(), nullable=False, server_default="0"),
+        sa.Column("volumes_size", sa.BigInteger(), nullable=False, server_default="0"),
+        sa.Column("build_cache_size", sa.BigInteger(), nullable=False, server_default="0"),
+        sa.Column("images_total", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("images_active", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("containers_count", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("volumes_count", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("host_id"),
+    )
+    op.create_table(
+        "docker_images",
+        sa.Column("host_id", sa.String(length=36), nullable=False),
+        sa.Column("image_id", sa.String(length=64), nullable=False),
+        sa.Column("repo_tag", sa.String(length=512), nullable=False, server_default="<none>"),
+        sa.Column("size", sa.BigInteger(), nullable=False, server_default="0"),
+        sa.Column("shared_size", sa.BigInteger(), nullable=False, server_default="0"),
+        sa.Column("containers", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("host_id", "image_id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("docker_images")
+    op.drop_table("docker_disk_usage")

plugins/docker/migrations/versions/docker_008_bigint_mem.py

@@ -0,0 +1,33 @@
+"""Widen memory byte columns to BIGINT
+
+docker_metrics.mem_usage_bytes and docker_containers.mem_usage_bytes /
+mem_limit_bytes were int4, which overflows for any container using >2.1 GB
+(asyncpg: "value out of int32 range"), failing the whole ingest batch. Widen to
+BIGINT — a safe in-place int4→int8 promotion, no data rewrite.
+
+Revision ID: docker_008_bigint_mem
+Revises: docker_007_disk_usage
+Create Date: 2026-06-20
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_008_bigint_mem"
+down_revision: Union[str, None] = "docker_007_disk_usage"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.alter_column("docker_metrics", "mem_usage_bytes", type_=sa.BigInteger())
+    op.alter_column("docker_containers", "mem_usage_bytes", type_=sa.BigInteger())
+    op.alter_column("docker_containers", "mem_limit_bytes", type_=sa.BigInteger())
+
+
+def downgrade() -> None:
+    # Narrowing back to int4 will error if any stored value exceeds 2^31 — that's
+    # the very condition this migration fixed, so downgrade is best-effort.
+    op.alter_column("docker_containers", "mem_limit_bytes", type_=sa.Integer())
+    op.alter_column("docker_containers", "mem_usage_bytes", type_=sa.Integer())
+    op.alter_column("docker_metrics", "mem_usage_bytes", type_=sa.Integer())

plugins/docker/migrations/versions/docker_009_container_logs.py

@@ -0,0 +1,48 @@
+"""Docker container logs table
+
+Adds docker_logs — one row per container log line, pushed by the host agent and
+folded into its metrics push. Time-series, host-scoped (container names are only
+unique within a host). Bounded by a per-container size+age ring in retention, so
+a chatty container keeps a shorter window rather than growing without limit.
+Additive create_table + twin indexes (viewer lookup by (host, container, ts);
+age-cutoff prune by ts).
+
+Revision ID: docker_009_container_logs
+Revises: docker_008_bigint_mem
+Create Date: 2026-07-19
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_009_container_logs"
+down_revision: Union[str, None] = "docker_008_bigint_mem"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "docker_logs",
+        sa.Column("id", sa.String(length=36), nullable=False),
+        sa.Column("host_id", sa.String(length=36), nullable=False),
+        sa.Column("container_name", sa.String(length=255), nullable=False),
+        sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("stream", sa.String(length=8), nullable=False, server_default="stdout"),
+        sa.Column("line", sa.Text(), nullable=False, server_default=""),
+        sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index("ix_docker_logs_host_id", "docker_logs", ["host_id"])
+    op.create_index("ix_docker_logs_container_name", "docker_logs", ["container_name"])
+    op.create_index("ix_docker_logs_host_container_time",
+                    "docker_logs", ["host_id", "container_name", "ts"])
+    op.create_index("ix_docker_logs_ts", "docker_logs", ["ts"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_docker_logs_ts", table_name="docker_logs")
+    op.drop_index("ix_docker_logs_host_container_time", table_name="docker_logs")
+    op.drop_index("ix_docker_logs_container_name", table_name="docker_logs")
+    op.drop_index("ix_docker_logs_host_id", table_name="docker_logs")
+    op.drop_table("docker_logs")

plugins/docker/models.py

@@ -2,24 +2,37 @@
 from __future__ import annotations
 import uuid
 from datetime import datetime, timezone
-from sqlalchemy import DateTime, Float, Integer, String, Text
+from sqlalchemy import (
+    BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text,
+    UniqueConstraint,
+)
 from sqlalchemy.orm import Mapped, mapped_column
 from steward.models.base import Base
 
 
 class DockerContainer(Base):
-    """Latest known state per container — upserted by name on each scrape."""
+    """Latest known state per container, scoped to the host that reported it.
+
+    Collection is per-host via the host agent, so container names are only
+    unique within a host — the natural key is (host_id, name). host_id is NOT
+    NULL: every container arrives through a host_agent ingest that resolves a
+    Host first. Deleting a host cascades its containers away.
+    """
     __tablename__ = "docker_containers"
 
-    # Container name is the stable natural key; container_id changes on recreation
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
+    )
     name: Mapped[str] = mapped_column(String(255), primary_key=True)
     container_id: Mapped[str] = mapped_column(String(64), nullable=False, default="")
     image: Mapped[str] = mapped_column(String(512), nullable=False, default="")
     status: Mapped[str] = mapped_column(String(32), nullable=False, default="unknown")
     # running | stopped | paused | exited | dead
     cpu_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
-    mem_usage_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
-    mem_limit_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    # BigInteger: a container's memory usage/limit routinely exceeds 2^31 bytes
+    # (~2.1 GB), which overflows int4 and fails the insert.
+    mem_usage_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+    mem_limit_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
     mem_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
     restart_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     ports_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
@@ -30,14 +43,35 @@ class DockerContainer(Base):
         default=lambda: datetime.now(timezone.utc),
     )
 
+    # ── Enrichment (agent ≥ 1.4.0) ────────────────────────────────────────────
+    # Health/exit/restart come from `docker inspect` (not the list endpoint);
+    # exit_code is only meaningful for stopped containers.
+    health: Mapped[str | None] = mapped_column(String(16), nullable=True)  # healthy|unhealthy|starting
+    exit_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    oom_killed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    # Grouping: compose project + swarm placement, read off container labels.
+    compose_project: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    service_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    task_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    node_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    # Cumulative-since-start I/O counters (BigInteger — they exceed 2^31 quickly).
+    net_rx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+    net_tx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+    blk_read_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+    blk_write_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+
 
 class DockerMetric(Base):
-    """Time-series CPU/memory per container — one row per scrape per running container."""
+    """Time-series CPU/memory per container — one row per sample per running
+    container, scoped to the reporting host."""
     __tablename__ = "docker_metrics"
 
     id: Mapped[str] = mapped_column(
         String(36), primary_key=True, default=lambda: str(uuid.uuid4())
     )
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
+    )
     container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
     scraped_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False,
@@ -46,4 +80,222 @@ class DockerMetric(Base):
     )
     cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
     mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
-    mem_usage_bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    # BigInteger: per-container memory usage exceeds 2^31 bytes (~2.1 GB) routinely.
+    mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
+
+    # Per-container history lookups filter on (host_id, container_name) then sort
+    # by time — a composite index keeps the rows() sparkline queries cheap.
+    __table_args__ = (
+        Index("ix_docker_metrics_host_container_time",
+              "host_id", "container_name", "scraped_at"),
+    )
+
+
+class DockerMetricHourly(Base):
+    """Hourly rollup of docker_metrics — avg cpu/mem per container per hour.
+
+    Raw per-sample rows (~2880/container/day at 30s) are pruned beyond a short
+    window; before deletion they're aggregated here so multi-day history stays
+    cheap to store and query. One row per (host, container, hour bucket); the
+    unique constraint lets retention upsert idempotently if it re-runs before the
+    raw rows are deleted. `bucket` is the hour-truncated sample time.
+    """
+    __tablename__ = "docker_metrics_hourly"
+
+    id: Mapped[str] = mapped_column(
+        String(36), primary_key=True, default=lambda: str(uuid.uuid4())
+    )
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
+    )
+    container_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+    mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+    mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
+    sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+
+    __table_args__ = (
+        # One bucket per container per host — the conflict target for the
+        # idempotent rollup upsert; doubles as the history-query index.
+        UniqueConstraint("host_id", "container_name", "bucket",
+                         name="uq_docker_metrics_hourly_bucket"),
+        Index("ix_docker_metrics_hourly_bucket", "bucket"),
+    )
+
+
+class DockerEvent(Base):
+    """Lifecycle events derived by diffing consecutive host snapshots.
+
+    The agent only reports current state, so Steward synthesises lifecycle by
+    comparing each new snapshot against the stored DockerContainer state for the
+    host: a container that appears → `start`; one that vanishes → `stop`; a
+    transition to a stopped status with a non-zero exit → `die`; an OOM-kill
+    flip → `oom`; a health status change → `health_change`. Scoped to the
+    reporting host (names are only unique within a host). `event` is a plain
+    string (no CHECK whitelist), matching how `status` is modelled on
+    DockerContainer — keeps the value set open without a migration per addition.
+    """
+    __tablename__ = "docker_events"
+
+    id: Mapped[str] = mapped_column(
+        String(36), primary_key=True, default=lambda: str(uuid.uuid4())
+    )
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
+    )
+    container_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    event: Mapped[str] = mapped_column(String(16), nullable=False)
+    # start | stop | die | oom | health_change
+    detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    # e.g. "exit 137", "healthy→unhealthy", image on start
+    at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False,
+        default=lambda: datetime.now(timezone.utc),
+    )
+
+    # Timeline lookups filter on (host_id, container_name) and sort by time;
+    # retention prunes by `at` alone — index both access paths.
+    __table_args__ = (
+        Index("ix_docker_events_host_container_time",
+              "host_id", "container_name", "at"),
+        Index("ix_docker_events_at", "at"),
+    )
+
+
+class DockerLog(Base):
+    """Container log lines pushed by the host agent — one row per line.
+
+    Time-series, scoped to the reporting host (container names are only unique
+    within a host, same identity as docker_metrics). The agent tails each running
+    container incrementally and folds new lines into its metrics push; `ts` is the
+    line's own Docker timestamp. Bounded by a per-container size+age ring
+    (retention), so a chatty container just keeps a shorter window rather than
+    growing without limit.
+    """
+    __tablename__ = "docker_logs"
+
+    id: Mapped[str] = mapped_column(
+        String(36), primary_key=True, default=lambda: str(uuid.uuid4())
+    )
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
+    ts: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False,
+        default=lambda: datetime.now(timezone.utc),
+    )
+    stream: Mapped[str] = mapped_column(String(8), nullable=False, default="stdout")
+    # stdout | stderr
+    line: Mapped[str] = mapped_column(Text, nullable=False, default="")
+
+    # Viewer filters on (host_id, container_name) and sorts by time; the ring
+    # prune walks the same key newest-first. A second index on `ts` alone serves
+    # the age-cutoff delete. Twin-index idiom, mirroring docker_events.
+    __table_args__ = (
+        Index("ix_docker_logs_host_container_time",
+              "host_id", "container_name", "ts"),
+        Index("ix_docker_logs_ts", "ts"),
+    )
+
+
+class DockerSwarmService(Base):
+    """A Swarm service as seen by a manager host: desired vs running replicas.
+
+    Manager-scoped — only a host that is a Swarm manager reports these, so a
+    single-manager cluster yields one row set keyed by that host. Service names
+    are unique within a swarm; we still key by (host_id, service_name) so two
+    managers reporting independently don't collide.
+    """
+    __tablename__ = "docker_swarm_services"
+
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
+    )
+    service_name: Mapped[str] = mapped_column(String(255), primary_key=True)
+    mode: Mapped[str] = mapped_column(String(16), nullable=False, default="replicated")
+    # replicated | global
+    desired: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    running: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    image: Mapped[str | None] = mapped_column(String(512), nullable=True)
+    # task→node placement of running replicas: [{"node_id", "running"}], JSON.
+    # A manager sees every task, so this captures cross-node placement that the
+    # local docker_containers rows (this host only) can't.
+    placement_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False,
+        default=lambda: datetime.now(timezone.utc),
+    )
+
+
+class DockerDiskUsage(Base):
+    """Per-host Docker disk usage summary from `/system/df` (current state).
+
+    One row per host (the agent re-reports the full summary each interval).
+    Reclaimable = bytes held by images no container references. Sizes are bytes
+    (BigInteger — image caches exceed 2^31 easily).
+    """
+    __tablename__ = "docker_disk_usage"
+
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
+    )
+    layers_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
+    images_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
+    images_reclaimable: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
+    containers_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
+    volumes_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
+    build_cache_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
+    images_total: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    images_active: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    containers_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    volumes_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False,
+        default=lambda: datetime.now(timezone.utc),
+    )
+
+
+class DockerImage(Base):
+    """Per-host image storage record (the heavy hitters from `/system/df`).
+
+    Keyed (host_id, image_id); `containers` is the reference count (0 ⇒ the
+    image's size is reclaimable). Replaced wholesale per host on each report.
+    """
+    __tablename__ = "docker_images"
+
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
+    )
+    image_id: Mapped[str] = mapped_column(String(64), primary_key=True)
+    repo_tag: Mapped[str] = mapped_column(String(512), nullable=False, default="<none>")
+    size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
+    shared_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
+    containers: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False,
+        default=lambda: datetime.now(timezone.utc),
+    )
+
+
+class DockerSwarmNode(Base):
+    """A Swarm node as seen by a manager host: role / availability / status."""
+    __tablename__ = "docker_swarm_nodes"
+
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
+    )
+    node_id: Mapped[str] = mapped_column(String(64), primary_key=True)
+    hostname: Mapped[str] = mapped_column(String(255), nullable=False, default="")
+    role: Mapped[str] = mapped_column(String(16), nullable=False, default="worker")
+    # manager | worker
+    availability: Mapped[str] = mapped_column(String(16), nullable=False, default="active")
+    # active | pause | drain
+    status: Mapped[str] = mapped_column(String(16), nullable=False, default="unknown")
+    # ready | down | unknown
+    leader: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False,
+        default=lambda: datetime.now(timezone.utc),
+    )

plugins/docker/plugin.yaml

@@ -1,6 +1,6 @@
 name: docker
-version: "1.0.0"
-description: "Docker container status, resource usage, restart tracking via Docker socket"
+version: "2.0.0"
+description: "Per-host Docker container status + resource usage, collected by the host agent"
 author: "Steward"
 license: "MIT"
 min_app_version: "0.1.0"
@@ -11,7 +11,6 @@ tags:
   - docker
   - infrastructure
 
-config:
-  socket_path: /var/run/docker.sock
-  scrape_interval_seconds: 60
-  include_stopped: false
+# No config: collection is agent-driven (the host agent reads each host's local
+# socket and pushes containers to the host_agent ingest). This plugin is pure
+# presentation + storage, so there's no socket path or scrape interval to tune.

plugins/docker/retention.py

@@ -0,0 +1,160 @@
+# plugins/docker/retention.py
+"""Bound Docker time-series growth: roll up old metrics, prune old rows.
+
+Published as the "docker.run_retention" capability (see __init__.setup) so the
+core cleanup task can drive it WITHOUT importing the docker models (same
+opportunistic-coupling pattern as docker.persist_host_samples). Runs inside the
+caller's open transaction; never opens or commits its own.
+
+The scaling concern is docker_metrics: ~2880 rows/container/day at a 30s sample.
+We keep raw samples for a short window, then aggregate everything older into
+hourly averages (docker_metrics_hourly) and delete the raw rows — so multi-day
+history stays cheap to store and query. docker_events is light but unbounded
+without a cutoff, so it gets a (longer) window too.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+
+def _hour_floor(dt: datetime) -> datetime:
+    """Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
+    return dt.replace(minute=0, second=0, microsecond=0)
+
+
+def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
+    """Hour-aligned boundary below which raw metrics get rolled up + deleted.
+
+    Aligning to the hour means we only ever roll up *whole* elapsed hours — a
+    bucket is never split across the keep/roll boundary, so re-running can't
+    produce a partial-then-complete duplicate for the same hour.
+    """
+    return _hour_floor(now - timedelta(days=raw_days))
+
+
+async def run_docker_retention(
+    session,
+    *,
+    events_days: int,
+    metrics_raw_days: int,
+    metrics_rollup_days: int,
+    logs_retention_days: int = 3,
+    logs_max_bytes_per_container: int = 5_000_000,
+    now: datetime | None = None,
+) -> dict:
+    """Roll up + prune Docker time-series. Returns a counts dict for logging.
+
+    1. Aggregate docker_metrics older than the (hour-aligned) raw window into
+       docker_metrics_hourly (avg cpu/mem per container per hour), upserting so a
+       re-run is idempotent, then delete those raw rows.
+    2. Prune rolled-up rows older than the rollup window.
+    3. Prune docker_events older than the events window.
+    4. Prune docker_logs with a per-container size+age ring (m79): drop lines
+       older than the age window, then keep only the newest ~cap bytes per
+       (host, container).
+    """
+    from datetime import timezone
+
+    from sqlalchemy import delete, func, select
+    from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+    from .models import DockerEvent, DockerLog, DockerMetric, DockerMetricHourly
+
+    if now is None:
+        now = datetime.now(timezone.utc)
+
+    rolled = rolled_rows = events_pruned = rollup_pruned = 0
+    logs_age_pruned = logs_size_pruned = 0
+
+    # ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
+    raw_cutoff = _rollup_cutoff(now, metrics_raw_days)
+    hour = func.date_trunc("hour", DockerMetric.scraped_at)
+    agg = (
+        select(
+            DockerMetric.host_id,
+            DockerMetric.container_name,
+            hour.label("bucket"),
+            func.avg(DockerMetric.cpu_pct).label("cpu_pct"),
+            func.avg(DockerMetric.mem_pct).label("mem_pct"),
+            func.avg(DockerMetric.mem_usage_bytes).label("mem_usage_bytes"),
+            func.count().label("sample_count"),
+        )
+        .where(DockerMetric.scraped_at < raw_cutoff)
+        .group_by(DockerMetric.host_id, DockerMetric.container_name, hour)
+    )
+    for r in (await session.execute(agg)).all():
+        stmt = (
+            pg_insert(DockerMetricHourly)
+            .values(
+                host_id=r.host_id,
+                container_name=r.container_name,
+                bucket=r.bucket,
+                cpu_pct=float(r.cpu_pct or 0.0),
+                mem_pct=float(r.mem_pct or 0.0),
+                mem_usage_bytes=int(r.mem_usage_bytes or 0),
+                sample_count=int(r.sample_count or 0),
+            )
+            .on_conflict_do_update(
+                constraint="uq_docker_metrics_hourly_bucket",
+                set_={
+                    "cpu_pct": float(r.cpu_pct or 0.0),
+                    "mem_pct": float(r.mem_pct or 0.0),
+                    "mem_usage_bytes": int(r.mem_usage_bytes or 0),
+                    "sample_count": int(r.sample_count or 0),
+                },
+            )
+        )
+        await session.execute(stmt)
+        rolled += 1
+        rolled_rows += int(r.sample_count or 0)
+    if rolled:
+        await session.execute(
+            delete(DockerMetric).where(DockerMetric.scraped_at < raw_cutoff)
+        )
+
+    # ── 2. Prune rolled-up rows beyond the rollup window ──
+    rollup_cutoff = now - timedelta(days=metrics_rollup_days)
+    res = await session.execute(
+        delete(DockerMetricHourly).where(DockerMetricHourly.bucket < rollup_cutoff)
+    )
+    rollup_pruned = res.rowcount or 0
+
+    # ── 3. Prune lifecycle events beyond the events window ──
+    events_cutoff = now - timedelta(days=events_days)
+    res = await session.execute(
+        delete(DockerEvent).where(DockerEvent.at < events_cutoff)
+    )
+    events_pruned = res.rowcount or 0
+
+    # ── 4. Container-log ring: age cutoff, then per-container byte cap (m79) ──
+    logs_cutoff = now - timedelta(days=logs_retention_days)
+    res = await session.execute(
+        delete(DockerLog).where(DockerLog.ts < logs_cutoff)
+    )
+    logs_age_pruned = res.rowcount or 0
+
+    # Size ring: per (host, container), sum line bytes newest-first; delete a row
+    # once its strictly-newer siblings already fill the cap. Using the EXCLUSIVE
+    # prefix (running total minus this row) means the newest row always survives,
+    # so a single line larger than the cap is never wiped out.
+    running = func.sum(func.length(DockerLog.line)).over(
+        partition_by=[DockerLog.host_id, DockerLog.container_name],
+        order_by=[DockerLog.ts.desc(), DockerLog.id.desc()],
+    )
+    prefix_excl = (running - func.length(DockerLog.line)).label("prefix_excl")
+    ranked = select(DockerLog.id, prefix_excl).subquery()
+    over_cap = select(ranked.c.id).where(
+        ranked.c.prefix_excl >= logs_max_bytes_per_container)
+    res = await session.execute(
+        delete(DockerLog).where(DockerLog.id.in_(over_cap))
+    )
+    logs_size_pruned = res.rowcount or 0
+
+    return {
+        "buckets_rolled": rolled,
+        "raw_rows_rolled": rolled_rows,
+        "rollup_pruned": rollup_pruned,
+        "events_pruned": events_pruned,
+        "logs_age_pruned": logs_age_pruned,
+        "logs_size_pruned": logs_size_pruned,
+    }

plugins/docker/routes.py

@@ -1,17 +1,79 @@
 # plugins/docker/routes.py
 from __future__ import annotations
 import json
-from quart import Blueprint, current_app, render_template, request
-from sqlalchemy import Integer, cast, func, select
+from datetime import datetime, timedelta, timezone
+
+from quart import (
+    Blueprint, current_app, jsonify, redirect, render_template, request,
+    session, url_for,
+)
+from sqlalchemy import func, select
 
 from steward.auth.middleware import require_role
 from steward.models.users import UserRole
-from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds
-from .models import DockerContainer, DockerMetric
+from steward.models.hosts import Host
+from steward.core.time_range import parse_range, DEFAULT_RANGE
+from .dedup import dedup_by_container_id
+from .swarm_view import build_swarm_services
+from .models import (
+    DockerContainer, DockerDiskUsage, DockerEvent, DockerImage, DockerLog,
+    DockerMetric, DockerSwarmNode, DockerSwarmService,
+)
 
 docker_bp = Blueprint("docker", __name__, template_folder="templates")
 
 
+def _error(status: int, code: str, detail: str | None = None):
+    body: dict = {"ok": False, "error": code}
+    if detail:
+        body["detail"] = detail
+    return jsonify(body), status
+
+
+def _prune_extra_vars(prune_target: str) -> dict:
+    """Extra-vars for the bundled docker_prune playbook. "Prune unused images"
+    means ALL unused (docker image prune -a), not just dangling; the full system
+    prune stays conservative (-f, no -a) per the operator's choice (m78). Pure
+    helper so the mapping is unit-tested without the request/DB stack."""
+    extra_vars: dict = {"prune_target": prune_target}
+    if prune_target == "images":
+        extra_vars["prune_all_images"] = True
+    return extra_vars
+
+
+def _human_bytes(n: int | None) -> str:
+    """Compact binary size string (e.g. '1.4 GiB', '512 MiB', '0 B')."""
+    if n is None:
+        return "—"
+    size = float(n)
+    for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+        if abs(size) < 1024.0 or unit == "TiB":
+            if unit == "B":
+                return f"{int(size)} {unit}"
+            return f"{size:.1f} {unit}"
+        size /= 1024.0
+    return f"{size:.1f} TiB"
+
+
+def _human_uptime(started_at: datetime | None) -> str | None:
+    """Compact 'how long running' string (e.g. '3d 4h', '5h 12m', '8m')."""
+    if started_at is None:
+        return None
+    if started_at.tzinfo is None:
+        started_at = started_at.replace(tzinfo=timezone.utc)
+    secs = int((datetime.now(timezone.utc) - started_at).total_seconds())
+    if secs < 0:
+        return None
+    d, rem = divmod(secs, 86400)
+    h, rem = divmod(rem, 3600)
+    m, _ = divmod(rem, 60)
+    if d:
+        return f"{d}d {h}h"
+    if h:
+        return f"{h}h {m}m"
+    return f"{m}m"
+
+
 def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
     if len(values) < 2:
         return f'<svg width="{width}" height="{height}"></svg>'
@@ -33,76 +95,150 @@ def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
     )
 
 
+def _bucket(values: list[float], target: int = 40) -> list[float]:
+    """Bucket-average a series down to ~target points (DB-agnostic, in Python).
+
+    Replaces the old SQL strftime() bucketing, which was SQLite-only and broke
+    on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of
+    rows — averaging into a readable point count keeps the sparkline's shape.
+    """
+    n = len(values)
+    if n <= target:
+        return values
+    size = (n + target - 1) // target
+    out: list[float] = []
+    for i in range(0, n, size):
+        chunk = values[i:i + size]
+        out.append(sum(chunk) / len(chunk))
+    return out
+
+
+async def _container_history(db, host_id: str, name: str, since) -> tuple[list, list]:
+    """Return (cpu_series, mem_series) sparkline-ready for one host's container."""
+    rows = (await db.execute(
+        select(DockerMetric.cpu_pct, DockerMetric.mem_pct)
+        .where(DockerMetric.host_id == host_id)
+        .where(DockerMetric.container_name == name)
+        .where(DockerMetric.scraped_at >= since)
+        .order_by(DockerMetric.scraped_at)
+    )).all()
+    cpu = _bucket([r.cpu_pct or 0.0 for r in rows])
+    mem = _bucket([r.mem_pct or 0.0 for r in rows])
+    return cpu, mem
+
+
 @docker_bp.get("/")
 @require_role(UserRole.viewer)
 async def index():
     poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
     current_range = request.args.get("range", DEFAULT_RANGE)
+    # Show the Swarm link only when a manager has actually reported topology, so
+    # non-swarm installs aren't offered an always-empty page.
+    async with current_app.db_sessionmaker() as db:
+        has_swarm = (await db.execute(
+            select(DockerSwarmService.host_id).limit(1))).first() is not None
+        has_disk = (await db.execute(
+            select(DockerDiskUsage.host_id).limit(1))).first() is not None
     return await render_template(
         "docker/index.html",
         poll_interval=poll_interval,
         current_range=current_range,
+        has_swarm=has_swarm,
+        has_disk=has_disk,
     )
 
 
+async def _host_map(db, host_ids: set[str]) -> dict[str, Host]:
+    if not host_ids:
+        return {}
+    return {
+        h.id: h for h in (await db.execute(
+            select(Host).where(Host.id.in_(host_ids)))).scalars().all()
+    }
+
+
 @docker_bp.get("/rows")
 @require_role(UserRole.viewer)
 async def rows():
-    """HTMX fragment: container list with status and resource sparklines."""
+    """HTMX fragment: swarm services (collapsed, cluster-complete) on top, then
+    non-swarm containers grouped by host with resource sparklines."""
     since, range_key = parse_range(request.args.get("range"))
-    b_secs = bucket_seconds(since)
-
-    bucket_col = (
-        cast(func.strftime('%s', DockerMetric.scraped_at), Integer) / b_secs
-    ).label("bucket")
 
     async with current_app.db_sessionmaker() as db:
-        # All known containers ordered by running first, then name
-        result = await db.execute(
-            select(DockerContainer)
-            .order_by(
-                # running first
+        containers = list((await db.execute(
+            select(DockerContainer).order_by(
                 (DockerContainer.status == "running").desc(),
                 DockerContainer.name,
             )
+        )).scalars())
+        services = list((await db.execute(select(DockerSwarmService))).scalars())
+        nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
+        hosts = await _host_map(
+            db,
+            {c.host_id for c in containers}
+            | {n.host_id for n in nodes} | {s.host_id for s in services},
         )
-        containers = list(result.scalars())
 
-        # Build per-container sparkline histories
-        histories: dict[str, list] = {}
-        for c in containers:
-            result = await db.execute(
-                select(
-                    func.avg(DockerMetric.cpu_pct).label("cpu_pct"),
-                    func.avg(DockerMetric.mem_pct).label("mem_pct"),
-                    bucket_col,
-                )
-                .where(DockerMetric.container_name == c.name)
-                .where(DockerMetric.scraped_at >= since)
-                .group_by(bucket_col)
-                .order_by(bucket_col)
-            )
-            histories[c.name] = result.all()
+        # Swarm tasks collapse into per-service rows (each replica labelled with
+        # its host, plus "ghost" replicas for tasks on agent-less nodes, drawn
+        # from the managers' placement). Non-swarm containers keep the host view.
+        node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
+        host_name = {h.id: h.name for h in hosts.values()}
+        swarm_view = build_swarm_services(containers, services, node_hostname, host_name)
 
-    running = sum(1 for c in containers if c.status == "running")
-    stopped = len(containers) - running
-
-    container_data = []
-    for c in containers:
-        hist = histories.get(c.name, [])
-        ports = json.loads(c.ports_json) if c.ports_json else []
-        container_data.append({
+        standalone = [c for c in containers if not c.service_name]
+        groups: dict[str, dict] = {}
+        for c in standalone:
+            g = groups.get(c.host_id)
+            if g is None:
+                host = hosts.get(c.host_id)
+                g = groups[c.host_id] = {
+                    "host": host,
+                    "host_id": c.host_id,
+                    "host_name": host.name if host else c.host_id,
+                    "containers": [], "running": 0, "stopped": 0,
+                }
+            cpu_hist, mem_hist = await _container_history(db, c.host_id, c.name, since)
+            g["containers"].append({
                 "container": c,
-            "ports": ports,
-            "sparkline_cpu": _sparkline([r.cpu_pct or 0 for r in hist]),
-            "sparkline_mem": _sparkline([r.mem_pct or 0 for r in hist]),
+                "ports": json.loads(c.ports_json) if c.ports_json else [],
+                "uptime": _human_uptime(c.started_at) if c.status == "running" else None,
+                "sparkline_cpu": _sparkline(cpu_hist),
+                "sparkline_mem": _sparkline(mem_hist),
             })
+            if c.status == "running":
+                g["running"] += 1
+            else:
+                g["stopped"] += 1
 
+    # Sub-group each host's containers by compose project, preserving the
+    # running-first ordering. Containers with no project fall into an unlabelled
+    # bucket rendered flat, so plain hosts look unchanged.
+    for g in groups.values():
+        subs: dict[str, dict] = {}
+        order: list[str] = []
+        for item in g["containers"]:
+            c = item["container"]
+            label = c.compose_project or None
+            key = label or ""
+            if key not in subs:
+                subs[key] = {"label": label, "containers": []}
+                order.append(key)
+            subs[key]["containers"].append(item)
+        g["subgroups"] = [subs[k] for k in order]
+        g["grouped"] = any(s["label"] for s in g["subgroups"])
+
+    host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
+    swarm_services = swarm_view["services"]
+    standalone_running = sum(g["running"] for g in host_groups)
+    ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"])
     return await render_template(
         "docker/rows.html",
-        container_data=container_data,
-        running=running,
-        stopped=stopped,
+        host_groups=host_groups,
+        swarm_services=swarm_services,
+        running=standalone_running + sum(s["running"] for s in swarm_services),
+        stopped=len(standalone) - standalone_running,
+        total=len(containers) + ghost_total,
         range_key=range_key,
     )
 
@@ -110,29 +246,62 @@ async def rows():
 @docker_bp.get("/widget")
 @require_role(UserRole.viewer)
 async def widget():
-    """HTMX dashboard widget: container status overview."""
+    """HTMX dashboard widget: container status overview, grouped by host."""
     show_stopped = request.args.get("show_stopped", "no") == "yes"
     widget_id = request.args.get("wid", "0")
+    since_24h = datetime.now(timezone.utc) - timedelta(hours=24)
 
     async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(DockerContainer)
-            .order_by(
+        all_containers = dedup_by_container_id(list((await db.execute(
+            select(DockerContainer).order_by(
                 (DockerContainer.status == "running").desc(),
                 DockerContainer.name,
             )
+        )).scalars()))
+        services = list((await db.execute(select(DockerSwarmService))).scalars())
+        nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
+        # Recent failures are more actionable than a static "stopped" tally. Count
+        # DISTINCT container names so the two managers' duplicate die/oom events
+        # don't double it.
+        failed_24h = (await db.execute(
+            select(func.count(func.distinct(DockerEvent.container_name)))
+            .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since_24h)
+        )).scalar() or 0
+        hosts = await _host_map(
+            db,
+            {c.host_id for c in all_containers}
+            | {n.host_id for n in nodes} | {s.host_id for s in services},
         )
-        all_containers = list(result.scalars())
 
-    running = [c for c in all_containers if c.status == "running"]
-    stopped = [c for c in all_containers if c.status != "running"]
-    display = all_containers if show_stopped else running
+    # Swarm tasks collapse into per-service rows (each replica tagged with its
+    # host); non-swarm containers stay grouped by host.
+    node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
+    host_name = {h.id: h.name for h in hosts.values()}
+    swarm_services = build_swarm_services(all_containers, services, node_hostname, host_name)["services"]
+
+    standalone = [c for c in all_containers if not c.service_name]
+    running = [c for c in standalone if c.status == "running"]
+    display = standalone if show_stopped else running
+
+    # Group for display so multi-host fleets read clearly; single-host stays flat.
+    groups: dict[str, dict] = {}
+    for c in display:
+        g = groups.setdefault(c.host_id, {
+            "host_id": c.host_id,
+            "host_name": host_name.get(c.host_id, c.host_id),
+            "containers": [],
+        })
+        g["containers"].append(c)
+    host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
 
     return await render_template(
         "docker/widget.html",
-        containers=display,
-        running_count=len(running),
-        stopped_count=len(stopped),
+        host_groups=host_groups,
+        multi_host=len(host_groups) > 1,
+        swarm_services=swarm_services,
+        running_count=len(running) + sum(s["running"] for s in swarm_services),
+        failed_24h=failed_24h,
+        total_count=len(all_containers) + len(swarm_services),
         show_stopped=show_stopped,
         widget_id=widget_id,
     )
@@ -141,20 +310,393 @@ async def widget():
 @docker_bp.get("/widget/resources")
 @require_role(UserRole.viewer)
 async def widget_resources():
-    """HTMX dashboard widget: CPU + memory usage for running containers."""
+    """HTMX dashboard widget: CPU + memory usage for the busiest containers."""
     limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
     widget_id = request.args.get("wid", "0")
 
     async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
+        # Dedup before the limit, else swarm tasks reported by both managers can
+        # fill the list with duplicates of the same container.
+        containers = dedup_by_container_id(list((await db.execute(
             select(DockerContainer)
             .where(DockerContainer.status == "running")
             .order_by(DockerContainer.cpu_pct.desc().nullslast())
-        )
-        containers = list(result.scalars())[:limit]
+        )).scalars()))[:limit]
+        hosts = await _host_map(db, {c.host_id for c in containers})
 
+    rows_data = [
+        {"c": c, "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id}
+        for c in containers
+    ]
     return await render_template(
         "docker/widget_resources.html",
-        containers=containers,
+        rows=rows_data,
+        multi_host=len({c.host_id for c in containers}) > 1,
         widget_id=widget_id,
     )
+
+
+@docker_bp.get("/host/<host_id>")
+@require_role(UserRole.viewer)
+async def host_panel(host_id: str):
+    """Per-host Docker fragment embedded on the core Hosts hub via HTMX.
+
+    Renders nothing when the host reports no containers, so hosts without Docker
+    don't carry an empty card.
+    """
+    async with current_app.db_sessionmaker() as db:
+        containers = list((await db.execute(
+            select(DockerContainer)
+            .where(DockerContainer.host_id == host_id)
+            .order_by(
+                (DockerContainer.status == "running").desc(),
+                DockerContainer.name,
+            )
+        )).scalars())
+
+    if not containers:
+        return ""
+    running = sum(1 for c in containers if c.status == "running")
+    return await render_template(
+        "docker/host_panel.html",
+        containers=containers,
+        running=running,
+        stopped=len(containers) - running,
+    )
+
+
+# Glyph + colour per lifecycle event, so the timeline reads at a glance.
+_EVENT_STYLE = {
+    "start":         ("▲", "var(--green)"),
+    "stop":          ("■", "var(--text-muted)"),
+    "die":           ("✕", "var(--red)"),
+    "oom":           ("☠", "var(--red)"),
+    "health_change": ("◐", "var(--orange)"),
+}
+
+
+@docker_bp.get("/container/<host_id>/<name>")
+@require_role(UserRole.viewer)
+async def container_detail(host_id: str, name: str):
+    """Full detail page for one container: facts, resource history, lifecycle."""
+    async with current_app.db_sessionmaker() as db:
+        c = await db.get(DockerContainer, (host_id, name))
+        host = await db.get(Host, host_id)
+        events = []
+        if c is not None:
+            events = list((await db.execute(
+                select(DockerEvent)
+                .where(DockerEvent.host_id == host_id)
+                .where(DockerEvent.container_name == name)
+                .order_by(DockerEvent.at.desc())
+                .limit(50)
+            )).scalars())
+
+    if c is None:
+        return await render_template(
+            "docker/container_detail.html",
+            container=None, host_id=host_id, name=name,
+        ), 404
+
+    return await render_template(
+        "docker/container_detail.html",
+        container=c,
+        host=host,
+        host_id=host_id,
+        name=name,
+        ports=json.loads(c.ports_json) if c.ports_json else [],
+        uptime=_human_uptime(c.started_at) if c.status == "running" else None,
+        net_rx=_human_bytes(c.net_rx_bytes), net_tx=_human_bytes(c.net_tx_bytes),
+        blk_read=_human_bytes(c.blk_read_bytes), blk_write=_human_bytes(c.blk_write_bytes),
+        events=[
+            {"event": e.event, "detail": e.detail, "at": e.at,
+             "glyph": _EVENT_STYLE.get(e.event, ("•", "var(--text-muted)"))[0],
+             "colour": _EVENT_STYLE.get(e.event, ("•", "var(--text-muted)"))[1]}
+            for e in events
+        ],
+        current_range=request.args.get("range", DEFAULT_RANGE),
+    )
+
+
+@docker_bp.get("/swarm")
+@require_role(UserRole.viewer)
+async def swarm():
+    """Swarm topology page: services with replica health, nodes, placement.
+
+    Swarm services/nodes are cluster-global — every manager's API returns the
+    same list — but each manager reports them independently (rows keyed by
+    host_id). So we group the reporting managers into swarms (managers that share
+    any node_id are the same cluster) and dedup within each: one row per service
+    (by name) and per node (by node_id), keeping the freshest. Without this, two
+    managers in one cluster list every service/node twice.
+    """
+    async with current_app.db_sessionmaker() as db:
+        services = list((await db.execute(
+            select(DockerSwarmService).order_by(DockerSwarmService.service_name)
+        )).scalars())
+        nodes = list((await db.execute(
+            select(DockerSwarmNode).order_by(
+                DockerSwarmNode.role, DockerSwarmNode.hostname)
+        )).scalars())
+        hosts = await _host_map(db, {s.host_id for s in services} | {n.host_id for n in nodes})
+
+    # ── Group reporting managers into swarms by shared node membership ──────────
+    manager_ids = {s.host_id for s in services} | {n.host_id for n in nodes}
+    nodes_by_mgr: dict[str, set[str]] = {}
+    for n in nodes:
+        nodes_by_mgr.setdefault(n.host_id, set()).add(n.node_id)
+
+    parent = {m: m for m in manager_ids}
+
+    def _find(x: str) -> str:
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]
+            x = parent[x]
+        return x
+
+    mgrs = list(manager_ids)
+    for i in range(len(mgrs)):
+        for j in range(i + 1, len(mgrs)):
+            a, b = mgrs[i], mgrs[j]
+            if nodes_by_mgr.get(a) and nodes_by_mgr.get(b) and (nodes_by_mgr[a] & nodes_by_mgr[b]):
+                parent[_find(a)] = _find(b)
+
+    swarm_members: dict[str, set[str]] = {}
+    for m in manager_ids:
+        swarm_members.setdefault(_find(m), set()).add(m)
+
+    # ── Dedup services (by name) + nodes (by node_id) within each swarm ─────────
+    swarms = []
+    for members in swarm_members.values():
+        svc_by_name: dict[str, DockerSwarmService] = {}
+        for s in services:
+            if s.host_id in members and (
+                s.service_name not in svc_by_name
+                or s.updated_at > svc_by_name[s.service_name].updated_at
+            ):
+                svc_by_name[s.service_name] = s
+        node_by_id: dict[str, DockerSwarmNode] = {}
+        for n in nodes:
+            if n.host_id in members and (
+                n.node_id not in node_by_id
+                or n.updated_at > node_by_id[n.node_id].updated_at
+            ):
+                node_by_id[n.node_id] = n
+
+        node_names = {nid: (nrow.hostname or nid) for nid, nrow in node_by_id.items()}
+        svc_dicts = []
+        for s in sorted(svc_by_name.values(), key=lambda s: s.service_name):
+            placement = []
+            for p in (json.loads(s.placement_json) if s.placement_json else []):
+                nid = p.get("node_id", "")
+                placement.append({
+                    "node": node_names.get(nid, nid[:12] or "?"),
+                    "running": p.get("running", 0),
+                })
+            svc_dicts.append({
+                "name": s.service_name, "mode": s.mode,
+                "running": s.running, "desired": s.desired, "image": s.image,
+                "healthy": s.running >= s.desired and s.desired > 0,
+                "degraded": 0 < s.running < s.desired,
+                "down": s.running == 0 and s.desired > 0,
+                "placement": placement,
+            })
+        node_rows = sorted(
+            node_by_id.values(), key=lambda n: (n.role, (n.hostname or n.node_id).lower())
+        )
+        manager_names = sorted(hosts[m].name if m in hosts else m for m in members)
+        swarms.append({
+            "managers": manager_names,
+            "services": svc_dicts,
+            "nodes": node_rows,
+        })
+
+    swarms.sort(key=lambda g: g["managers"][0].lower() if g["managers"] else "")
+    return await render_template("docker/swarm.html", swarms=swarms)
+
+
+@docker_bp.get("/disk")
+@require_role(UserRole.viewer)
+async def disk():
+    """Image/disk usage page: reclaimable space, per-image sizes, stopped count.
+
+    Admins can reclaim space per host via the prune buttons, which fire the
+    audited bundled prune playbook through Ansible (m78) — the collection agent
+    itself stays read-only.
+    """
+    from steward.core.capabilities import has_capability
+    from steward.models.ansible_inventory import AnsibleTarget
+
+    async with current_app.db_sessionmaker() as db:
+        summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
+        images = list((await db.execute(
+            select(DockerImage).order_by(DockerImage.size.desc())
+        )).scalars())
+        # Stopped-container count per host, surfaced alongside reclaimable space.
+        stopped_rows = (await db.execute(
+            select(DockerContainer.host_id)
+            .where(DockerContainer.status != "running")
+        )).scalars()
+        stopped_by_host: dict[str, int] = {}
+        for hid in stopped_rows:
+            stopped_by_host[hid] = stopped_by_host.get(hid, 0) + 1
+        hosts = await _host_map(db, {s.host_id for s in summaries})
+        # Which hosts have a linked Ansible target — gates the prune buttons
+        # (a target is required to route the playbook run to that host).
+        host_ids = {s.host_id for s in summaries}
+        target_by_host: dict[str, str] = {}
+        if host_ids:
+            for hid, tid in (await db.execute(
+                select(AnsibleTarget.host_id, AnsibleTarget.id)
+                .where(AnsibleTarget.host_id.in_(host_ids))
+            )).all():
+                target_by_host[hid] = tid
+
+    images_by_host: dict[str, list] = {}
+    for im in images:
+        images_by_host.setdefault(im.host_id, []).append({
+            "repo_tag": im.repo_tag, "size": _human_bytes(im.size),
+            "shared": _human_bytes(im.shared_size), "containers": im.containers,
+            "reclaimable": im.containers == 0,
+        })
+
+    host_groups = [
+        {
+            "host_id": s.host_id,
+            "host_name": hosts[s.host_id].name if s.host_id in hosts else s.host_id,
+            "summary": {
+                "images_total": s.images_total, "images_active": s.images_active,
+                "images_size": _human_bytes(s.images_size),
+                "images_reclaimable": _human_bytes(s.images_reclaimable),
+                "layers_size": _human_bytes(s.layers_size),
+                "containers_count": s.containers_count,
+                "containers_size": _human_bytes(s.containers_size),
+                "volumes_count": s.volumes_count,
+                "volumes_size": _human_bytes(s.volumes_size),
+                "build_cache_size": _human_bytes(s.build_cache_size),
+            },
+            "stopped": stopped_by_host.get(s.host_id, 0),
+            "images": images_by_host.get(s.host_id, []),
+            "has_target": s.host_id in target_by_host,
+        }
+        for s in summaries
+    ]
+    host_groups.sort(key=lambda g: g["host_name"].lower())
+    return await render_template(
+        "docker/disk.html", host_groups=host_groups,
+        ansible_available=has_capability("ansible.run_playbook"),
+    )
+
+
+@docker_bp.post("/disk/<host_id>/prune")
+@require_role(UserRole.admin)
+async def disk_prune(host_id: str):
+    """Reclaim Docker disk on a host by firing the bundled prune playbook via
+    Ansible (audited, admin-gated) — the collection agent stays read-only (m78).
+    `target` selects the scope: containers | images | system.
+    """
+    from steward.core.capabilities import has_capability, invoke_capability
+    from steward.ansible.sources import BUILTIN_SOURCE_NAME
+    from steward.models.ansible_inventory import AnsibleTarget
+
+    if not has_capability("ansible.run_playbook"):
+        return _error(400, "ansible_unavailable", "Ansible is not available")
+
+    form = await request.form
+    prune_target = (form.get("target", "") or "").strip()
+    if prune_target not in ("containers", "images", "system"):
+        return _error(400, "bad_target", "Unknown prune target")
+
+    async with current_app.db_sessionmaker() as db:
+        target = (await db.execute(
+            select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
+        )).scalar_one_or_none()
+    if target is None:
+        return _error(400, "no_target",
+                      "Link an Ansible target to this host before pruning")
+
+    # extra_vars_map outranks the playbook's own `vars:` defaults.
+    extra_vars = _prune_extra_vars(prune_target)
+
+    actor_role = UserRole(session.get("user_role", "viewer"))
+    run, _source, err = await invoke_capability(
+        "ansible.run_playbook", actor_role,
+        current_app._get_current_object(),  # type: ignore[attr-defined]
+        source_name=BUILTIN_SOURCE_NAME,
+        playbook_path="maintenance/docker_prune.yml",
+        inventory_scope=f"steward:target:{target.id}",
+        params={"extra_vars_map": extra_vars},
+        triggered_by=session.get("user_id"),
+    )
+    if err:
+        return _error(400, "prune_failed", err)
+    return redirect(url_for("ansible.run_detail", run_id=run.id))
+
+
+@docker_bp.get("/container/<host_id>/<name>/history")
+@require_role(UserRole.viewer)
+async def container_history(host_id: str, name: str):
+    """HTMX fragment: CPU + memory sparklines for the selected time range."""
+    since, range_key = parse_range(request.args.get("range"))
+    async with current_app.db_sessionmaker() as db:
+        cpu_hist, mem_hist = await _container_history(db, host_id, name, since)
+    return await render_template(
+        "docker/_container_history.html",
+        sparkline_cpu=_sparkline(cpu_hist, width=320, height=48),
+        sparkline_mem=_sparkline(mem_hist, width=320, height=48),
+        have_data=len(cpu_hist) >= 2,
+        range_key=range_key,
+    )
+
+
+# Newest lines returned per fetch — the retained window is bounded by the ring,
+# but a container can still hold thousands of lines; cap what one fetch renders.
+_LOG_TAIL_DEFAULT = 500
+
+
+async def _query_logs(db, host_id: str, name: str, stream: str, query: str,
+                      limit: int) -> list:
+    """The recent retained lines for one container, newest-first, optionally
+    filtered by stream and a case-insensitive substring.
+
+    Newest-first is deliberate: the viewer replaces the list on each poll, which
+    resets scroll to the top — so the latest lines stay visible without any
+    scroll handling, and older lines are a scroll away.
+    """
+    stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line)
+            .where(DockerLog.host_id == host_id)
+            .where(DockerLog.container_name == name))
+    if stream in ("stdout", "stderr"):
+        stmt = stmt.where(DockerLog.stream == stream)
+    if query:
+        stmt = stmt.where(DockerLog.line.ilike(f"%{query}%"))
+    stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
+    return (await db.execute(stmt)).all()
+
+
+@docker_bp.get("/container/<host_id>/<name>/logs")
+@require_role(UserRole.viewer)
+async def container_logs(host_id: str, name: str):
+    """Full log-viewer page for one container (lines load via an HTMX fragment)."""
+    async with current_app.db_sessionmaker() as db:
+        host = await db.get(Host, host_id)
+    return await render_template(
+        "docker/container_logs.html", host=host, host_id=host_id, name=name,
+    )
+
+
+@docker_bp.get("/container/<host_id>/<name>/logs/lines")
+@require_role(UserRole.viewer)
+async def container_logs_lines(host_id: str, name: str):
+    """HTMX fragment: the recent retained log lines, stream + text filtered.
+    Polled every few seconds by the viewer for near-live follow."""
+    stream = request.args.get("stream", "all")
+    query = (request.args.get("q") or "").strip()
+    async with current_app.db_sessionmaker() as db:
+        rows = await _query_logs(db, host_id, name, stream, query, _LOG_TAIL_DEFAULT)
+    return await render_template(
+        "docker/_container_logs_lines.html",
+        lines=[{"ts": r.ts, "stream": r.stream, "line": r.line} for r in rows],
+        name=name, tail=_LOG_TAIL_DEFAULT,
+        filtered=bool(query) or stream in ("stdout", "stderr"),
+    )

plugins/docker/scheduler.py

@@ -1,93 +0,0 @@
-# plugins/docker/scheduler.py
-from __future__ import annotations
-import json
-import logging
-from datetime import datetime, timezone
-
-from steward.core.scheduler import ScheduledTask
-from steward.core.alerts import record_metric
-
-logger = logging.getLogger(__name__)
-
-
-def make_task(app) -> ScheduledTask:
-    interval = int(
-        app.config["PLUGINS"]["docker"].get("scrape_interval_seconds", 60)
-    )
-
-    async def scrape():
-        await _do_scrape(app)
-
-    return ScheduledTask(
-        name="docker_scrape",
-        coro_factory=scrape,
-        interval_seconds=interval,
-        run_on_startup=True,
-    )
-
-
-async def _do_scrape(app) -> None:
-    from .scraper import scrape_docker
-    from .models import DockerContainer, DockerMetric
-
-    cfg = app.config["PLUGINS"]["docker"]
-    socket_path = cfg.get("socket_path", "/var/run/docker.sock")
-    include_stopped = bool(cfg.get("include_stopped", False))
-
-    try:
-        containers = await scrape_docker(socket_path, include_stopped)
-    except ConnectionError as exc:
-        logger.error("Docker scrape failed: %s", exc)
-        return
-    except Exception:
-        logger.exception("Docker scrape error")
-        return
-
-    now = datetime.now(timezone.utc)
-
-    async with app.db_sessionmaker() as session:
-        async with session.begin():
-            for c in containers:
-                # Upsert container state
-                existing = await session.get(DockerContainer, c["name"])
-                if existing is None:
-                    existing = DockerContainer(name=c["name"])
-                    session.add(existing)
-                existing.container_id = c["container_id"]
-                existing.image = c["image"]
-                existing.status = c["status"]
-                existing.cpu_pct = c["cpu_pct"]
-                existing.mem_usage_bytes = c["mem_usage_bytes"]
-                existing.mem_limit_bytes = c["mem_limit_bytes"]
-                existing.mem_pct = c["mem_pct"]
-                existing.restart_count = c["restart_count"]
-                existing.ports_json = json.dumps(c["ports"])
-                existing.started_at = c["started_at"]
-                existing.scraped_at = now
-
-                # Time-series metric (running containers only)
-                if c["status"] == "running" and c["cpu_pct"] is not None:
-                    session.add(DockerMetric(
-                        container_name=c["name"],
-                        scraped_at=now,
-                        cpu_pct=c["cpu_pct"],
-                        mem_pct=c["mem_pct"] or 0.0,
-                        mem_usage_bytes=c["mem_usage_bytes"] or 0,
-                    ))
-
-                    # Feed alert pipeline
-                    await record_metric(
-                        session=session,
-                        source_module="docker",
-                        resource_name=c["name"],
-                        metric_name="cpu_pct",
-                        value=c["cpu_pct"],
-                    )
-                    if c["mem_pct"] is not None:
-                        await record_metric(
-                            session=session,
-                            source_module="docker",
-                            resource_name=c["name"],
-                            metric_name="mem_pct",
-                            value=c["mem_pct"],
-                        )

plugins/docker/scraper.py

@@ -1,139 +0,0 @@
-# plugins/docker/scraper.py
-"""Docker API client via Unix socket."""
-from __future__ import annotations
-import asyncio
-import logging
-from datetime import datetime, timezone
-
-import httpx
-
-logger = logging.getLogger(__name__)
-
-
-def _short_id(container_id: str) -> str:
-    return container_id[:12] if container_id else ""
-
-
-def _calc_cpu_pct(stats: dict) -> float:
-    """Calculate CPU % from a Docker stats snapshot (one-shot)."""
-    try:
-        cpu = stats.get("cpu_stats", {})
-        precpu = stats.get("precpu_stats", {})
-        cpu_delta = (
-            cpu["cpu_usage"]["total_usage"] - precpu["cpu_usage"]["total_usage"]
-        )
-        sys_delta = cpu.get("system_cpu_usage", 0) - precpu.get("system_cpu_usage", 0)
-        num_cpus = cpu.get("online_cpus") or len(
-            cpu["cpu_usage"].get("percpu_usage") or [None]
-        )
-        if sys_delta <= 0 or cpu_delta < 0:
-            return 0.0
-        return round((cpu_delta / sys_delta) * num_cpus * 100.0, 2)
-    except (KeyError, TypeError, ZeroDivisionError):
-        return 0.0
-
-
-def _calc_mem(stats: dict) -> tuple[int, int, float]:
-    """Return (usage_bytes, limit_bytes, mem_pct) from Docker stats."""
-    try:
-        mem = stats.get("memory_stats", {})
-        usage = mem.get("usage", 0)
-        limit = mem.get("limit", 0)
-        # Subtract page cache for working set (cgroup v1: "cache", v2: "inactive_file")
-        cache = mem.get("stats", {}).get("cache", 0) or \
-                mem.get("stats", {}).get("inactive_file", 0)
-        actual = max(0, usage - cache)
-        pct = round(actual / limit * 100.0, 2) if limit > 0 else 0.0
-        return actual, limit, pct
-    except (KeyError, TypeError):
-        return 0, 0, 0.0
-
-
-def _parse_ports(ports: list) -> list[dict]:
-    """Normalise Docker port bindings to a compact list."""
-    result = []
-    for p in (ports or []):
-        if p.get("PublicPort"):
-            result.append({
-                "host_port": p["PublicPort"],
-                "container_port": p["PrivatePort"],
-                "protocol": p.get("Type", "tcp"),
-            })
-    return result
-
-
-async def scrape_docker(socket_path: str, include_stopped: bool) -> list[dict]:
-    """
-    Fetch all containers + resource stats from the Docker daemon.
-    Returns a list of normalised dicts ready for the scheduler to persist.
-    Raises ConnectionError if the socket is unreachable.
-    """
-    transport = httpx.AsyncHTTPTransport(uds=socket_path)
-    try:
-        async with httpx.AsyncClient(
-            transport=transport,
-            base_url="http://docker",
-            timeout=10.0,
-        ) as client:
-            resp = await client.get(
-                "/containers/json",
-                params={"all": "true" if include_stopped else "false"},
-            )
-            resp.raise_for_status()
-            containers = resp.json()
-
-            async def _get_stats(c: dict) -> tuple[dict, dict | None]:
-                if c.get("State") != "running":
-                    return c, None
-                try:
-                    r = await client.get(
-                        f"/containers/{c['Id']}/stats",
-                        params={"stream": "false", "one-shot": "true"},
-                        timeout=5.0,
-                    )
-                    return c, r.json()
-                except Exception as exc:
-                    logger.debug(
-                        "Stats unavailable for %s: %s", _short_id(c.get("Id", "")), exc
-                    )
-                    return c, None
-
-            pairs = await asyncio.gather(*[_get_stats(c) for c in containers])
-
-    except httpx.ConnectError as exc:
-        raise ConnectionError(
-            f"Cannot connect to Docker socket at {socket_path}: {exc}"
-        ) from exc
-
-    results = []
-    for container, stats in pairs:
-        names = container.get("Names") or []
-        name = names[0].lstrip("/") if names else _short_id(container.get("Id", ""))
-
-        cpu_pct = mem_usage = mem_limit = mem_pct = None
-        if stats is not None:
-            cpu_pct = _calc_cpu_pct(stats)
-            mem_usage, mem_limit, mem_pct = _calc_mem(stats)
-
-        # Created is a Unix epoch int from /containers/json
-        created_ts = container.get("Created")
-        started_at = (
-            datetime.fromtimestamp(created_ts, tz=timezone.utc)
-            if isinstance(created_ts, (int, float)) else None
-        )
-
-        results.append({
-            "name": name,
-            "container_id": _short_id(container.get("Id", "")),
-            "image": container.get("Image", ""),
-            "status": container.get("State", "unknown"),
-            "cpu_pct": cpu_pct,
-            "mem_usage_bytes": mem_usage,
-            "mem_limit_bytes": mem_limit,
-            "mem_pct": mem_pct,
-            "restart_count": 0,  # would require /inspect; left as 0 for now
-            "ports": _parse_ports(container.get("Ports", [])),
-            "started_at": started_at,
-        })
-
-    return results

plugins/docker/swarm_view.py

@@ -0,0 +1,140 @@
+"""Model-free builder for the swarm-aware container view.
+
+Merges two sources into one service-grouped, cluster-complete picture:
+  • docker_containers — real container rows, collected LOCAL-per-node, so each
+    carries the host (= node) it runs on plus its swarm service/node labels.
+  • DockerSwarmService.placement_json — the managers' cluster-wide task→node
+    counts, which include tasks on nodes that run no Steward agent (and so have
+    no container row of their own).
+
+For every swarm service we list the real replicas we have detail for, then add
+"ghost" replicas for the remaining placement count on each node (agent-less
+nodes). Non-swarm containers pass through grouped by host, unchanged.
+
+Kept model-free (no ORM imports) so it's unit-testable and safe to import in
+tests without the plugin-loader "table already defined" gotcha.
+"""
+from __future__ import annotations
+
+import json
+
+
+def _service_state(running: int, desired: int) -> str:
+    if desired > 0 and running >= desired:
+        return "healthy"
+    if running > 0:
+        return "degraded"
+    return "down"
+
+
+def _placement(service) -> list[dict]:
+    raw = getattr(service, "placement_json", None) or "[]"
+    try:
+        data = json.loads(raw)
+    except (ValueError, TypeError):
+        return []
+    return [p for p in data if isinstance(p, dict)]
+
+
+def build_swarm_services(containers, services, node_hostname: dict, host_name: dict) -> dict:
+    """Return {"services": [...], "standalone": [...]}.
+
+    services[i] = {name, mode, desired, running, state, replicas:[...]}
+      real replica  = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
+                       health, restart_count}
+      ghost replica = {ghost:True, host, count}   # placement with no local row
+    standalone[i] = {host_id, host, containers:[...]}  # non-swarm, by host
+    """
+    containers = list(containers)
+    swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
+    standalone_cs = [c for c in containers if not getattr(c, "service_name", None)]
+
+    # Dedup service rows by name (every manager reports the same cluster-global
+    # set); keep the first — placement is identical across managers.
+    svc_by_name: dict[str, object] = {}
+    for s in services:
+        name = getattr(s, "service_name", None)
+        if name and name not in svc_by_name:
+            svc_by_name[name] = s
+
+    # Real containers grouped by service name.
+    cs_by_service: dict[str, list] = {}
+    for c in swarm_cs:
+        cs_by_service.setdefault(c.service_name, []).append(c)
+
+    def _host_of(c):
+        return (host_name.get(getattr(c, "host_id", None))
+                or node_hostname.get(getattr(c, "node_id", None))
+                or getattr(c, "host_id", None) or "?")
+
+    out_services = []
+    for name in sorted(set(svc_by_name) | set(cs_by_service)):
+        svc = svc_by_name.get(name)
+        reals = cs_by_service.get(name, [])
+
+        replicas = []
+        for c in sorted(reals, key=lambda c: (_host_of(c).lower(), c.name)):
+            replicas.append({
+                "ghost": False,
+                "host": _host_of(c),
+                "host_id": getattr(c, "host_id", None),
+                "name": c.name,
+                "status": getattr(c, "status", "unknown"),
+                "cpu_pct": getattr(c, "cpu_pct", None),
+                "mem_pct": getattr(c, "mem_pct", None),
+                "health": getattr(c, "health", None),
+                "restart_count": getattr(c, "restart_count", 0) or 0,
+            })
+
+        # Real running replicas per node, to subtract from placement.
+        real_running_by_node: dict[str, int] = {}
+        for c in reals:
+            if (getattr(c, "status", "") or "").lower() == "running":
+                nid = getattr(c, "node_id", None)
+                if nid:
+                    real_running_by_node[nid] = real_running_by_node.get(nid, 0) + 1
+
+        ghosts = []
+        for p in _placement(svc) if svc is not None else []:
+            nid = p.get("node_id", "")
+            placed = int(p.get("running", 0) or 0)
+            ghost = placed - real_running_by_node.get(nid, 0)
+            if ghost > 0:
+                ghosts.append({
+                    "ghost": True,
+                    "host": node_hostname.get(nid) or (nid[:12] if nid else "?"),
+                    "count": ghost,
+                })
+        ghosts.sort(key=lambda g: g["host"].lower())
+        replicas.extend(ghosts)
+
+        running = getattr(svc, "running", None) if svc is not None else None
+        desired = getattr(svc, "desired", None) if svc is not None else None
+        # No service row (manager didn't report it): infer from what we can see.
+        if running is None:
+            running = sum(1 for r in replicas
+                          if (not r["ghost"] and (r["status"] or "").lower() == "running"))
+        if desired is None:
+            desired = len(replicas)
+
+        out_services.append({
+            "name": name,
+            "mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
+            "running": running,
+            "desired": desired,
+            "state": _service_state(int(running or 0), int(desired or 0)),
+            "replicas": replicas,
+        })
+
+    # Non-swarm containers grouped by host, running-first order preserved.
+    groups: dict[str, dict] = {}
+    for c in standalone_cs:
+        hid = getattr(c, "host_id", None)
+        g = groups.get(hid)
+        if g is None:
+            g = groups[hid] = {"host_id": hid, "host": host_name.get(hid) or hid or "?",
+                               "containers": []}
+        g["containers"].append(c)
+    standalone = sorted(groups.values(), key=lambda g: g["host"].lower())
+
+    return {"services": out_services, "standalone": standalone}

plugins/docker/templates/docker/_container_history.html

@@ -0,0 +1,17 @@
+{# docker/_container_history.html — CPU/mem sparklines for the selected range #}
+{% if have_data %}
+<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:1.5rem;">
+  <div>
+    <div class="section-title" style="margin-bottom:0.4rem;">CPU %</div>
+    {{ sparkline_cpu | safe }}
+  </div>
+  <div>
+    <div class="section-title" style="margin-bottom:0.4rem;">Memory %</div>
+    {{ sparkline_mem | safe }}
+  </div>
+</div>
+{% else %}
+<div style="color:var(--text-muted);font-size:0.85rem;">
+  Not enough samples in this range yet — history fills in as the agent reports.
+</div>
+{% endif %}

plugins/docker/templates/docker/_container_logs_lines.html

@@ -0,0 +1,22 @@
+{# docker/_container_logs_lines.html — log-line fragment, HTMX-polled (m79).
+   {{ l.line }} is auto-escaped by Jinja, so log content can't inject markup. #}
+{% if lines %}
+{% for l in lines %}
+<div style="display:flex;gap:0.6rem;white-space:pre-wrap;word-break:break-word;">
+  <span style="color:var(--text-dim);flex-shrink:0;" title="{{ l.ts }}">{{ l.ts.strftime("%m-%d %H:%M:%S") }}</span>
+  <span style="flex-shrink:0;width:3.2rem;color:{% if l.stream == 'stderr' %}var(--red){% else %}var(--text-muted){% endif %};">{{ l.stream }}</span>
+  <span style="flex:1;min-width:0;{% if l.stream == 'stderr' %}color:var(--text);{% endif %}">{{ l.line }}</span>
+</div>
+{% endfor %}
+{% else %}
+<div style="color:var(--text-muted);padding:1rem 0.25rem;">
+  {% if filtered %}
+  No lines match the current filter.
+  {% else %}
+  No logs collected yet for <code>{{ name }}</code>. Lines appear within a few
+  seconds of the container writing to stdout/stderr — unless it's on the log
+  exclude list or log collection is turned off in
+  <a href="/settings/thresholds/">Settings → Thresholds &amp; Retention</a>.
+  {% endif %}
+</div>
+{% endif %}

plugins/docker/templates/docker/container_detail.html

@@ -0,0 +1,111 @@
+{# docker/container_detail.html — full detail page for one container #}
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs %}
+{% block title %}{{ name }} — Docker — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
+{% block content %}
+
+{% if container is none %}
+<div class="card" style="text-align:center;padding:3rem;">
+  <div style="font-weight:600;margin-bottom:0.4rem;">Container not found</div>
+  <div style="color:var(--text-muted);font-size:0.9rem;">
+    No container named <code>{{ name }}</code> is currently reported for this host.
+    It may have been removed, or the host agent hasn't reported recently.
+  </div>
+</div>
+{% else %}
+
+{# ── Header ──────────────────────────────────────────────────────────────── #}
+<div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
+  <span class="dot {% if container.status == 'running' %}dot-up{% elif container.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
+  <h1 class="page-title" style="margin-bottom:0;">{{ container.name }}</h1>
+  {% if container.health == 'healthy' %}<span style="font-size:0.72rem;color:var(--green);border:1px solid var(--green);border-radius:3px;padding:0.05rem 0.4rem;">healthy</span>
+  {% elif container.health == 'unhealthy' %}<span style="font-size:0.72rem;color:var(--red);border:1px solid var(--red);border-radius:3px;padding:0.05rem 0.4rem;">unhealthy</span>
+  {% elif container.health == 'starting' %}<span style="font-size:0.72rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0.05rem 0.4rem;">starting</span>{% endif %}
+</div>
+<div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
+  {{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %}
+  {% if host %} · on <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %}
+  · <a href="/plugins/docker/container/{{ host_id }}/{{ name }}/logs">View logs</a>
+</div>
+
+{# ── Facts grid ──────────────────────────────────────────────────────────── #}
+{% macro fact(label, value, colour="") %}
+<div class="card" style="margin-bottom:0;">
+  <div class="section-title" style="margin-bottom:0.3rem;">{{ label }}</div>
+  <div style="font-size:0.95rem;{% if colour %}color:{{ colour }};{% endif %}font-family:ui-monospace,monospace;">{{ value }}</div>
+</div>
+{% endmacro %}
+
+<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
+  {{ fact("CPU", "%.1f%%" | format(container.cpu_pct) if container.cpu_pct is not none else "—") }}
+  {{ fact("Memory", "%.1f%%" | format(container.mem_pct) if container.mem_pct is not none else "—") }}
+  {{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }}
+  {{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—",
+          "var(--red)" if (container.exit_code is not none and container.exit_code != 0) else "") }}
+  {{ fact("Net in", net_rx) }}
+  {{ fact("Net out", net_tx) }}
+  {{ fact("Block read", blk_read) }}
+  {{ fact("Block write", blk_write) }}
+</div>
+
+{# ── Image / placement ───────────────────────────────────────────────────── #}
+<div class="card" style="margin-bottom:1.5rem;">
+  <div style="display:grid;grid-template-columns:max-content 1fr;gap:0.4rem 1.25rem;font-size:0.86rem;">
+    <span style="color:var(--text-muted);">Image</span>
+    <span style="font-family:ui-monospace,monospace;word-break:break-all;">{{ container.image or "—" }}</span>
+    {% if container.compose_project %}
+    <span style="color:var(--text-muted);">Compose project</span><span>{{ container.compose_project }}</span>
+    {% endif %}
+    {% if container.service_name %}
+    <span style="color:var(--text-muted);">Swarm service</span><span>{{ container.service_name }}</span>
+    {% endif %}
+    {% if container.node_id %}
+    <span style="color:var(--text-muted);">Node</span><span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>
+    {% endif %}
+    {% if ports %}
+    <span style="color:var(--text-muted);">Ports</span>
+    <span style="font-family:ui-monospace,monospace;">{% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %}</span>
+    {% endif %}
+  </div>
+</div>
+
+{# ── Resource history (range-toggled) ────────────────────────────────────── #}
+<div class="card" style="margin-bottom:1.5rem;">
+  <div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
+    <h3 class="section-title" style="margin-bottom:0;">Resource history</h3>
+    {% include "_time_range.html" %}
+  </div>
+  <div id="container-history"
+       hx-get="/plugins/docker/container/{{ host_id }}/{{ name }}/history"
+       hx-trigger="load, rangeChange from:body"
+       hx-include="#time-range"
+       hx-swap="innerHTML">
+    <div style="color:var(--text-muted);font-size:0.85rem;">Loading…</div>
+  </div>
+</div>
+
+{# ── Lifecycle timeline ──────────────────────────────────────────────────── #}
+<div class="card">
+  <h3 class="section-title" style="margin-bottom:0.75rem;">Lifecycle</h3>
+  {% if events %}
+  <div style="display:grid;gap:0.5rem;">
+    {% for e in events %}
+    <div style="display:flex;align-items:baseline;gap:0.6rem;font-size:0.85rem;">
+      <span style="color:{{ e.colour }};width:1rem;text-align:center;flex-shrink:0;">{{ e.glyph }}</span>
+      <span style="font-weight:500;width:6.5rem;flex-shrink:0;">{{ e.event }}</span>
+      <span style="color:var(--text-muted);flex:1;min-width:0;">{{ e.detail or "" }}</span>
+      <span style="color:var(--text-dim);font-size:0.78rem;white-space:nowrap;" title="{{ e.at }}">{{ e.at.strftime("%Y-%m-%d %H:%M") }}</span>
+    </div>
+    {% endfor %}
+  </div>
+  {% else %}
+  <div style="color:var(--text-muted);font-size:0.85rem;">
+    No lifecycle events recorded yet. Start/stop/health changes appear here as the
+    agent reports them over time.
+  </div>
+  {% endif %}
+</div>
+
+{% endif %}
+{% endblock %}

plugins/docker/templates/docker/container_logs.html

@@ -0,0 +1,51 @@
+{# docker/container_logs.html — per-container log viewer (m79) #}
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs %}
+{% block title %}Logs — {{ name }} — Docker — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([
+  ("Docker", "/plugins/docker/"),
+  (name, "/plugins/docker/container/" ~ host_id ~ "/" ~ name),
+  ("Logs", "")]) }}{% endblock %}
+{% block content %}
+
+<div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
+  <h1 class="page-title" style="margin-bottom:0;">{{ name }}</h1>
+  <span style="font-size:0.9rem;color:var(--text-muted);">logs</span>
+</div>
+<div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1rem;">
+  Recent lines (newest first) collected by the host agent{% if host %} on
+  <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %} —
+  updated every few seconds.
+  <a href="/plugins/docker/container/{{ host_id }}/{{ name }}">← back to container</a>
+</div>
+
+{# ── Controls: stream filter + text search (drive the fragment via HTMX) ───── #}
+<form id="log-controls" onsubmit="return false;"
+      style="display:flex;gap:0.6rem;align-items:center;flex-wrap:wrap;margin-bottom:0.6rem;">
+  <label style="font-size:0.8rem;color:var(--text-muted);display:flex;align-items:center;gap:0.35rem;">
+    Stream
+    <select name="stream" style="padding:0.25rem 0.4rem;">
+      <option value="all">all</option>
+      <option value="stdout">stdout</option>
+      <option value="stderr">stderr</option>
+    </select>
+  </label>
+  <input type="search" name="q" placeholder="Filter lines…" autocomplete="off"
+         aria-label="Filter log lines"
+         style="flex:1;min-width:180px;padding:0.3rem 0.5rem;">
+</form>
+
+<div class="card-flush">
+  <div id="log-lines"
+       hx-get="/plugins/docker/container/{{ host_id }}/{{ name }}/logs/lines"
+       hx-trigger="load, every 5s, change from:#log-controls, keyup changed delay:400ms from:#log-controls"
+       hx-include="#log-controls"
+       hx-swap="innerHTML"
+       role="log" aria-live="polite" tabindex="0"
+       style="max-height:70vh;overflow:auto;padding:0.5rem 0.75rem;
+              font-family:ui-monospace,monospace;font-size:0.8rem;line-height:1.5;">
+    <div style="color:var(--text-muted);">Loading…</div>
+  </div>
+</div>
+
+{% endblock %}

plugins/docker/templates/docker/disk.html

@@ -0,0 +1,120 @@
+{# docker/disk.html — image/disk usage: reclaimable space, per-image sizes #}
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs %}
+{% block title %}Disk — Docker — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Image & disk usage", "")]) }}{% endblock %}
+{% block content %}
+
+<h1 class="page-title" style="margin-bottom:0.4rem;">Image &amp; disk usage</h1>
+<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
+  Reclaimable = space held by images no container references. Admins can prune
+  per host below; each action runs an audited Ansible playbook on that host.
+</p>
+
+{% if host_groups %}
+{% for g in host_groups %}
+<div style="margin-bottom:2rem;">
+  <div style="font-weight:600;font-size:0.95rem;margin-bottom:0.75rem;">{{ g.host_name }}</div>
+
+  {# ── Summary stats ────────────────────────────────────────────────────── #}
+  <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1rem;">
+    <div class="card" style="margin-bottom:0;">
+      <div class="section-title" style="margin-bottom:0.3rem;">Reclaimable</div>
+      <span class="stat-val" style="color:{% if g.summary.images_reclaimable != '0 B' %}var(--orange){% else %}var(--green){% endif %};">{{ g.summary.images_reclaimable }}</span>
+    </div>
+    <div class="card" style="margin-bottom:0;">
+      <div class="section-title" style="margin-bottom:0.3rem;">Images</div>
+      <span class="stat-val">{{ g.summary.images_size }}</span>
+      <div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.images_active }}/{{ g.summary.images_total }} in use</div>
+    </div>
+    <div class="card" style="margin-bottom:0;">
+      <div class="section-title" style="margin-bottom:0.3rem;">Stopped</div>
+      <span class="stat-val" style="{% if g.stopped %}color:var(--text-muted){% endif %};">{{ g.stopped }}</span>
+    </div>
+    <div class="card" style="margin-bottom:0;">
+      <div class="section-title" style="margin-bottom:0.3rem;">Writable layers</div>
+      <span class="stat-val">{{ g.summary.containers_size }}</span>
+      <div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.containers_count }} containers</div>
+    </div>
+    <div class="card" style="margin-bottom:0;">
+      <div class="section-title" style="margin-bottom:0.3rem;">Volumes</div>
+      <span class="stat-val">{{ g.summary.volumes_size }}</span>
+      <div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.volumes_count }} volumes</div>
+    </div>
+    <div class="card" style="margin-bottom:0;">
+      <div class="section-title" style="margin-bottom:0.3rem;">Build cache</div>
+      <span class="stat-val">{{ g.summary.build_cache_size }}</span>
+    </div>
+  </div>
+
+  {# ── Cleanup actions (admin only; audited Ansible prune run) ───────────── #}
+  {% if session.user_role == 'admin' %}
+  <div style="margin-bottom:1rem;">
+    {% if ansible_available and g.has_target %}
+    <div style="display:flex;flex-wrap:wrap;gap:0.5rem;align-items:center;">
+      <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
+            data-msg="Remove all STOPPED containers on {{ g.host_name|e }}? This cannot be undone."
+            onsubmit="return confirm(this.dataset.msg);">
+        <input type="hidden" name="target" value="containers">
+        <button type="submit" class="btn btn-sm">Prune stopped containers</button>
+      </form>
+      <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
+            data-msg="Remove ALL unused images on {{ g.host_name|e }} (docker image prune -a)? Any image not used by a container is deleted."
+            onsubmit="return confirm(this.dataset.msg);">
+        <input type="hidden" name="target" value="images">
+        <button type="submit" class="btn btn-sm">Prune unused images</button>
+      </form>
+      <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
+            data-msg="Run a full system prune on {{ g.host_name|e }}? Removes stopped containers, unused networks, dangling images and build cache."
+            onsubmit="return confirm(this.dataset.msg);">
+        <input type="hidden" name="target" value="system">
+        <button type="submit" class="btn btn-sm btn-danger">System prune…</button>
+      </form>
+    </div>
+    <div style="font-size:0.72rem;color:var(--text-muted);margin-top:0.4rem;">
+      Reclaimed space appears on the next agent sample.
+    </div>
+    {% elif not ansible_available %}
+    <div style="font-size:0.74rem;color:var(--text-muted);">Cleanup needs the Ansible runner (currently unavailable).</div>
+    {% else %}
+    <div style="font-size:0.74rem;color:var(--text-muted);">
+      Link an <a href="/hosts/{{ g.host_id }}">Ansible target</a> to this host to enable prune actions.
+    </div>
+    {% endif %}
+  </div>
+  {% endif %}
+
+  {# ── Per-image table ──────────────────────────────────────────────────── #}
+  <div class="card-flush">
+    <table class="table">
+      <thead>
+        <tr><th>Image</th><th>Size</th><th>Shared</th><th>Containers</th></tr>
+      </thead>
+      <tbody>
+        {% for im in g.images %}
+        <tr>
+          <td style="font-size:0.84rem;font-family:ui-monospace,monospace;max-width:340px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
+            {{ im.repo_tag }}
+            {% if im.reclaimable %}<span title="not referenced by any container" style="font-size:0.68rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0 0.3rem;margin-left:0.4rem;">reclaimable</span>{% endif %}
+          </td>
+          <td style="font-family:ui-monospace,monospace;font-size:0.86rem;">{{ im.size }}</td>
+          <td style="font-family:ui-monospace,monospace;font-size:0.86rem;color:var(--text-muted);">{{ im.shared }}</td>
+          <td style="font-size:0.86rem;color:{% if im.containers %}var(--text){% else %}var(--text-muted){% endif %};">{{ im.containers }}</td>
+        </tr>
+        {% else %}
+        <tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No images reported.</td></tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </div>
+</div>
+{% endfor %}
+{% else %}
+<div class="card" style="text-align:center;padding:3rem;">
+  <div style="color:var(--text-muted);font-size:0.9rem;">
+    No disk usage reported yet. The host agent reports image/disk usage from
+    <code>docker system df</code> on hosts running Docker.
+  </div>
+</div>
+{% endif %}
+{% endblock %}

plugins/docker/templates/docker/host_panel.html

@@ -0,0 +1,22 @@
+{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub #}
+<div class="card">
+  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;">
+    <h3 class="section-title" style="margin-bottom:0;">Docker</h3>
+    <span style="font-size:0.78rem;color:var(--text-muted);">
+      {{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %}
+      <a href="/plugins/docker/" style="margin-left:0.6rem;color:var(--text-muted);">All →</a>
+    </span>
+  </div>
+  <div style="display:grid;gap:2px;">
+    {% for c in containers %}
+    <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.85rem;padding:0.2rem 0;">
+      <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
+      <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;" title="{{ c.image }}">{{ c.name }}</a>
+      <span style="font-size:0.74rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
+        {% if c.cpu_pct is not none %}{{ "%.1f" | format(c.cpu_pct) }}%{% endif %}
+        {% if c.mem_pct is not none %} · {{ "%.1f" | format(c.mem_pct) }}%{% endif %}
+      </span>
+    </div>
+    {% endfor %}
+  </div>
+</div>

plugins/docker/templates/docker/index.html

@@ -2,7 +2,15 @@
 {% block title %}Docker — Steward{% endblock %}
 {% block content %}
 <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1.5rem;gap:1rem;flex-wrap:wrap;">
+  <div style="display:flex;align-items:baseline;gap:1rem;">
     <h1 class="page-title" style="margin-bottom:0;">Docker</h1>
+    {% if has_swarm %}
+    <a href="/plugins/docker/swarm" style="font-size:0.85rem;color:var(--accent);text-decoration:none;">Swarm →</a>
+    {% endif %}
+    {% if has_disk %}
+    <a href="/plugins/docker/disk" style="font-size:0.85rem;color:var(--accent);text-decoration:none;">Disk →</a>
+    {% endif %}
+  </div>
   {% include "_time_range.html" %}
 </div>
 

plugins/docker/templates/docker/rows.html

@@ -1,4 +1,4 @@
-{# docker/rows.html — HTMX fragment for the Docker main page #}
+{# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #}
 
 {# ── Summary strip ─────────────────────────────────────────────────────────── #}
 <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(140px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
@@ -12,13 +12,77 @@
   </div>
   <div class="card" style="margin-bottom:0;">
     <div class="section-title" style="margin-bottom:0.35rem;">Total</div>
-    <span class="stat-val">{{ container_data | length }}</span>
+    <span class="stat-val">{{ total }}</span>
+  </div>
+  {% if swarm_services %}
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.35rem;">Services</div>
+    <span class="stat-val">{{ swarm_services | length }}</span>
+  </div>
+  {% endif %}
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.35rem;">Hosts</div>
+    <span class="stat-val">{{ host_groups | length }}</span>
   </div>
 </div>
 
-{# ── Container table ─────────────────────────────────────────────────────── #}
-{% if container_data %}
-<div class="card-flush">
+{# ── Swarm services: collapsed per-service, each replica labelled with its host.
+   Ghost replicas (tasks on nodes with no Steward agent) come from the managers'
+   placement data so the picture is cluster-complete. ──────────────────────── #}
+{% if swarm_services %}
+<div class="card-flush" style="margin-bottom:1.5rem;">
+  <div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
+    <span style="font-weight:600;font-size:0.95rem;">Swarm services</span>
+    <a href="/plugins/docker/swarm" style="font-size:0.78rem;color:var(--text-muted);">Topology →</a>
+  </div>
+  <div style="padding:0.5rem 0.75rem;display:grid;gap:0.85rem;">
+    {% for s in swarm_services %}
+    <div>
+      <div style="display:flex;align-items:center;gap:0.55rem;flex-wrap:wrap;margin-bottom:0.35rem;">
+        <span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
+        <span style="font-weight:600;font-size:0.9rem;">{{ s.name }}</span>
+        <span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
+        <span style="font-size:0.78rem;color:{% if s.state == 'healthy' %}var(--green){% elif s.state == 'degraded' %}var(--orange){% else %}var(--red){% endif %};font-variant-numeric:tabular-nums;">
+          {{ s.running }}/{{ s.desired }} running
+        </span>
+      </div>
+      <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(210px,1fr));gap:0.4rem;padding-left:1.1rem;">
+        {% for r in s.replicas %}
+        {% if r.ghost %}
+        <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:5px;padding:0.3rem 0.55rem;" title="Running on a node with no Steward agent — detail unavailable">
+          <span class="dot dot-warn" style="opacity:0.6;"></span>
+          <span style="font-weight:500;">{{ r.host }}</span>
+          <span style="color:var(--text-dim);margin-left:auto;">{{ r.count }} · no agent</span>
+        </div>
+        {% else %}
+        <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px solid var(--border);border-radius:5px;padding:0.3rem 0.55rem;">
+          <span class="dot {% if r.status == 'running' %}dot-up{% elif r.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
+          <a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" style="font-weight:500;color:inherit;text-decoration:none;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" title="{{ r.name }}">{{ r.host }}</a>
+          {% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;">{{ "%.0f" | format(r.cpu_pct) }}%</span>{% endif %}
+        </div>
+        {% endif %}
+        {% endfor %}
+      </div>
+    </div>
+    {% endfor %}
+  </div>
+</div>
+{% endif %}
+
+{# ── Container tables, one per host ───────────────────────────────────────── #}
+{% if host_groups %}
+{% for g in host_groups %}
+<div class="card-flush" style="margin-bottom:1.5rem;">
+  <div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
+    {% if g.host %}
+    <a href="/hosts/{{ g.host.id }}" style="font-weight:600;font-size:0.95rem;color:inherit;text-decoration:none;">{{ g.host.name }}</a>
+    {% else %}
+    <span style="font-weight:600;font-size:0.95rem;">{{ g.host_name }}</span>
+    {% endif %}
+    <span style="font-size:0.78rem;color:var(--text-muted);">
+      {{ g.running }} running{% if g.stopped %} · {{ g.stopped }} stopped{% endif %}
+    </span>
+  </div>
   <table class="table">
     <thead>
       <tr>
@@ -32,15 +96,35 @@
       </tr>
     </thead>
     <tbody>
-      {% for item in container_data %}
+      {% for sub in g.subgroups %}
+      {% if g.grouped and sub.label %}
+      <tr><td colspan="7" style="padding:0.4rem 0.75rem 0.2rem;font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;font-weight:600;">{{ sub.label }}</td></tr>
+      {% endif %}
+      {% for item in sub.containers %}
       {% set c = item.container %}
       <tr>
         <td>
           <div style="display:flex;align-items:center;gap:0.5rem;">
             <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
             <div>
-              <div style="font-weight:500;font-size:0.9rem;">{{ c.name }}</div>
-              <div style="font-size:0.73rem;color:var(--text-muted);">{{ c.status }}</div>
+              <div style="font-weight:500;font-size:0.9rem;">
+                <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="color:inherit;text-decoration:none;border-bottom:1px dotted var(--border-mid);">{{ c.name }}</a>
+                {% if c.health == 'healthy' %}<span title="healthy" style="color:var(--green);font-size:0.7rem;">●</span>
+                {% elif c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);font-size:0.7rem;">●</span>
+                {% elif c.health == 'starting' %}<span title="health: starting" style="color:var(--orange);font-size:0.7rem;">◐</span>{% endif %}
+              </div>
+              <div style="font-size:0.73rem;color:var(--text-muted);">
+                {{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
+                {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
+                  · <span style="color:var(--red);">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
+                {% endif %}
+                {% if c.restart_count %} · <span title="restart count" style="color:var(--orange);">⟳{{ c.restart_count }}</span>{% endif %}
+              </div>
+              {% if c.service_name or c.compose_project %}
+              <div style="font-size:0.68rem;color:var(--text-dim);margin-top:0.1rem;">
+                {{ c.service_name or c.compose_project }}
+              </div>
+              {% endif %}
             </div>
           </div>
         </td>
@@ -74,13 +158,17 @@
         <td>{{ item.sparkline_mem | safe }}</td>
       </tr>
       {% endfor %}
+      {% endfor %}
     </tbody>
   </table>
 </div>
-{% else %}
+{% endfor %}
+{% elif not swarm_services %}
 <div class="card" style="text-align:center;padding:3rem;">
   <div style="color:var(--text-muted);font-size:0.9rem;">
-    No containers found. Make sure the Docker socket is accessible and the plugin is configured correctly.
+    No containers reported yet. Containers are collected by the Steward host agent —
+    deploy the agent to a host running Docker (Hosts → the host → agent panel) and
+    its containers will appear here under that host.
   </div>
 </div>
 {% endif %}

plugins/docker/templates/docker/swarm.html

@@ -0,0 +1,87 @@
+{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement #}
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs %}
+{% block title %}Swarm — Docker — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Swarm", "")]) }}{% endblock %}
+{% block content %}
+
+<h1 class="page-title" style="margin-bottom:1.5rem;">Swarm</h1>
+
+{% if swarms %}
+{% for g in swarms %}
+<div style="margin-bottom:2rem;">
+  <div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.75rem;flex-wrap:wrap;">
+    <span style="font-weight:600;font-size:0.95rem;">Swarm</span>
+    <span style="font-size:0.78rem;color:var(--text-muted);">
+      reported by {{ g.managers | length }} manager{{ 's' if g.managers | length != 1 }} · {{ g.managers | join(", ") }}
+    </span>
+  </div>
+
+  {# ── Services ─────────────────────────────────────────────────────────── #}
+  <div class="card-flush" style="margin-bottom:1rem;">
+    <table class="table">
+      <thead>
+        <tr>
+          <th>Service</th><th>Mode</th><th>Replicas</th><th>Image</th><th>Placement</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for s in g.services %}
+        <tr>
+          <td style="font-weight:500;font-size:0.9rem;">{{ s.name }}</td>
+          <td style="font-size:0.82rem;color:var(--text-muted);">{{ s.mode }}</td>
+          <td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
+            <span style="color:{% if s.healthy %}var(--green){% elif s.down %}var(--red){% elif s.degraded %}var(--orange){% else %}var(--text-muted){% endif %};">
+              {{ s.running }}/{{ s.desired }}
+            </span>
+          </td>
+          <td style="font-size:0.8rem;color:var(--text-muted);max-width:220px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.image or "—" }}</td>
+          <td style="font-size:0.78rem;color:var(--text-muted);">
+            {% for p in s.placement %}{{ p.node }} ({{ p.running }}){% if not loop.last %}, {% endif %}{% else %}—{% endfor %}
+          </td>
+        </tr>
+        {% else %}
+        <tr><td colspan="5" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No services in this swarm.</td></tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </div>
+
+  {# ── Nodes ────────────────────────────────────────────────────────────── #}
+  <div class="card-flush">
+    <table class="table">
+      <thead>
+        <tr><th>Node</th><th>Role</th><th>Availability</th><th>Status</th></tr>
+      </thead>
+      <tbody>
+        {% for n in g.nodes %}
+        <tr>
+          <td style="font-weight:500;font-size:0.9rem;">
+            {{ n.hostname or n.node_id }}
+            {% if n.leader %}<span title="cluster leader" style="font-size:0.68rem;color:var(--accent);border:1px solid var(--accent);border-radius:3px;padding:0 0.3rem;margin-left:0.3rem;">leader</span>{% endif %}
+          </td>
+          <td style="font-size:0.82rem;color:var(--text-muted);">{{ n.role }}</td>
+          <td style="font-size:0.82rem;color:{% if n.availability == 'active' %}var(--text){% else %}var(--orange){% endif %};">{{ n.availability }}</td>
+          <td style="font-size:0.82rem;">
+            <span class="dot {% if n.status == 'ready' %}dot-up{% else %}dot-down{% endif %}"></span>
+            <span style="color:{% if n.status == 'ready' %}var(--green){% else %}var(--red){% endif %};">{{ n.status }}</span>
+          </td>
+        </tr>
+        {% else %}
+        <tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No nodes reported.</td></tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </div>
+</div>
+{% endfor %}
+{% else %}
+<div class="card" style="text-align:center;padding:3rem;">
+  <div style="color:var(--text-muted);font-size:0.9rem;">
+    No Swarm topology reported. A Steward host agent running on a Swarm
+    <strong>manager</strong> reports services, nodes, and task placement here —
+    workers and non-swarm hosts contribute only their local containers.
+  </div>
+</div>
+{% endif %}
+{% endblock %}

plugins/docker/templates/docker/widget.html

@@ -1,28 +1,59 @@
-{# docker/widget.html — dashboard widget: container status overview #}
-{% if not containers and running_count == 0 and stopped_count == 0 %}
-<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No containers found.</div>
+{# docker/widget.html — dashboard widget: container status overview, by host #}
+{% if total_count == 0 %}
+<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No containers reported yet.</p>
 {% else %}
 
-<div style="display:flex;gap:1rem;margin-bottom:0.65rem;">
+<div style="display:flex;gap:1.25rem;margin-bottom:0.65rem;">
   <div>
     <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--green);">{{ running_count }}</span>
     <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">running</span>
   </div>
-  {% if stopped_count %}
-  <div>
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--text-muted);">{{ stopped_count }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">stopped</span>
+  <div title="Distinct containers with a die or OOM event in the last 24h">
+    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:{% if failed_24h %}var(--red){% else %}var(--text-muted){% endif %};">{{ failed_24h }}</span>
+    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">failed (24h)</span>
   </div>
-  {% endif %}
 </div>
 
+{# Swarm services — collapsed, each with its replicas' host chips (dashed = a
+   node with no Steward agent, count-only from placement). #}
+{% if swarm_services %}
+<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">Swarm services</div>
+<div style="display:grid;gap:0.4rem;margin-bottom:0.4rem;">
+  {% for s in swarm_services %}
+  <div>
+    <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
+      <span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
+      <span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
+      <span style="color:var(--text-muted);font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;">{{ s.running }}/{{ s.desired }}</span>
+    </div>
+    <div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">
+      {% for r in s.replicas %}
+      {% if r.ghost %}
+      <span title="no Steward agent on this node — count from swarm placement" style="font-size:0.68rem;color:var(--text-dim);background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }} ×{{ r.count }}</span>
+      {% else %}
+      <span style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }}</span>
+      {% endif %}
+      {% endfor %}
+    </div>
+  </div>
+  {% endfor %}
+</div>
+{% endif %}
+
+{% for g in host_groups %}
+{% if multi_host %}
+<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">{{ g.host_name }}</div>
+{% endif %}
 <div style="display:grid;gap:2px;">
-  {% for c in containers %}
+  {% for c in g.containers %}
   <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
     <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
-    <span style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
-      {{ c.name }}
-    </span>
+    <a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;">
+      {{ c.name }}{% if c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);"> ●</span>{% elif c.health == 'healthy' %}<span title="healthy" style="color:var(--green);"> ●</span>{% endif %}{% if c.restart_count %}<span title="restarts" style="color:var(--orange);font-size:0.7rem;"> ⟳{{ c.restart_count }}</span>{% endif %}
+    </a>
+    {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
+    <span title="last exit code" style="font-size:0.7rem;color:var(--red);flex-shrink:0;">exit {{ c.exit_code }}</span>
+    {% endif %}
     {% if c.cpu_pct is not none %}
     <span style="font-size:0.72rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
       {{ "%.1f" | format(c.cpu_pct) }}%
@@ -31,5 +62,6 @@
   </div>
   {% endfor %}
 </div>
+{% endfor %}
 
 {% endif %}

plugins/docker/templates/docker/widget_resources.html

@@ -1,35 +1,28 @@
-{# docker/widget_resources.html — dashboard widget: CPU + memory bars for running containers #}
-{% if not containers %}
+{# docker/widget_resources.html — dashboard widget: the busiest running containers
+   by CPU, each with labeled CPU + memory utilisation bars. #}
+{% if not rows %}
 <div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No running containers.</div>
 {% else %}
-<div style="display:grid;gap:0.5rem;">
-  {% for c in containers %}
+{% macro bar(label, pct, warn, crit) %}
+<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;">
+  <span style="font-size:0.62rem;color:var(--text-dim);width:1.9rem;flex-shrink:0;letter-spacing:0.03em;">{{ label }}</span>
+  <div style="flex:1;height:6px;background:var(--bg-elevated);border-radius:3px;overflow:hidden;">
+    <div style="height:100%;border-radius:3px;width:{{ [pct, 100] | min }}%;
+                background:{% if pct > crit %}var(--red){% elif pct > warn %}var(--orange){% else %}var(--accent){% endif %};
+                transition:width 0.4s;"></div>
+  </div>
+  <span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;width:3rem;text-align:right;flex-shrink:0;">{{ "%.1f" | format(pct) }}%</span>
+</div>
+{% endmacro %}
+<div style="display:grid;gap:0.6rem;">
+  {% for r in rows %}
+  {% set c = r.c %}
   <div>
-    <div style="display:flex;justify-content:space-between;align-items:baseline;margin-bottom:0.15rem;">
-      <span style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;max-width:55%;">
-        {{ c.name }}
-      </span>
-      <span style="font-size:0.72rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;margin-left:0.5rem;">
-        {% if c.cpu_pct is not none %}CPU {{ "%.1f" | format(c.cpu_pct) }}%{% endif %}
-        {% if c.mem_pct is not none %} · Mem {{ "%.1f" | format(c.mem_pct) }}%{% endif %}
-      </span>
+    <div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;">
+      {{ c.name }}{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
     </div>
-    {% if c.cpu_pct is not none %}
-    <div style="height:3px;background:var(--bg-elevated);border-radius:2px;margin-bottom:2px;">
-      <div style="height:100%;border-radius:2px;width:{{ [c.cpu_pct, 100] | min }}%;
-                  background:{% if c.cpu_pct > 80 %}var(--red){% elif c.cpu_pct > 50 %}var(--orange){% else %}var(--accent){% endif %};
-                  transition:width 0.4s;">
-      </div>
-    </div>
-    {% endif %}
-    {% if c.mem_pct is not none %}
-    <div style="height:3px;background:var(--bg-elevated);border-radius:2px;">
-      <div style="height:100%;border-radius:2px;width:{{ [c.mem_pct, 100] | min }}%;
-                  background:{% if c.mem_pct > 90 %}var(--red){% elif c.mem_pct > 70 %}var(--orange){% else %}var(--green){% endif %};
-                  transition:width 0.4s;">
-      </div>
-    </div>
-    {% endif %}
+    {% if c.cpu_pct is not none %}{{ bar("CPU", c.cpu_pct, 50, 80) }}{% endif %}
+    {% if c.mem_pct is not none %}{{ bar("MEM", c.mem_pct, 70, 90) }}{% endif %}
   </div>
   {% endfor %}
 </div>

plugins/host_agent/metrics_query.py

@@ -0,0 +1,112 @@
+"""Host-metric read helpers (latest snapshot + bucketed history).
+
+Kept separate from routes.py so it imports only the core PluginMetric model — not
+the host_agent ORM models — which lets integration tests import these helpers
+without tripping the plugin-loader's double-registration ("Table already
+defined") guard.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import func, or_, select
+
+from steward.core.time_range import bucket_seconds
+from steward.models.metrics import PluginMetric, PluginMetricHourly
+
+SOURCE_MODULE = "host_agent"
+
+# Host-level metrics charted on the detail page (sub-resources are shown as
+# current-value lists, not time series, to keep the page readable).
+HISTORY_METRICS = (
+    "cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m",
+    "net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
+    "temp_c_max", "psi_mem_some_avg10",
+)
+
+
+async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]:
+    """{resource_name: {metric: value}} — latest sample per (resource, metric) for
+    a host + its sub-resources.
+
+    DISTINCT ON picks the newest row per group in one index-ordered pass over
+    ix_plugin_metrics_module_resource_metric_recorded, instead of a GROUP-BY-max
+    subquery self-joined back to the table (two passes over the whole history).
+    """
+    rows = (await session.execute(
+        select(PluginMetric)
+        .where(
+            PluginMetric.source_module == SOURCE_MODULE,
+            or_(
+                PluginMetric.resource_name == host_name,
+                PluginMetric.resource_name.like(host_name + ":%"),
+            ),
+        )
+        .distinct(PluginMetric.resource_name, PluginMetric.metric_name)
+        .order_by(
+            PluginMetric.resource_name,
+            PluginMetric.metric_name,
+            PluginMetric.recorded_at.desc(),
+        )
+    )).scalars().all()
+    out: dict[str, dict[str, float]] = {}
+    for r in rows:
+        out.setdefault(r.resource_name, {})[r.metric_name] = r.value
+    return out
+
+
+async def _history_for_host(session, host_name: str, since, *, raw_days: int = 7) -> dict[str, list]:
+    """{metric: [[epoch_ms, avg_value], …]} host-level series since `since`.
+
+    Retention rolls raw plugin_metrics older than `raw_days` into hourly averages
+    (plugin_metrics_hourly) and deletes the raw rows. So we read the rollup for the
+    part of the range older than that boundary and raw (bucket-averaged in SQL,
+    capped to ≤1h to match the rollup) for the recent part — never shipping raw
+    samples to Python. The two windows don't overlap, so appending hourly-then-raw
+    keeps each metric's series time-ordered. Epoch-ms x feeds a linear chart axis.
+    """
+    now = datetime.now(timezone.utc)
+    raw_cutoff = (now - timedelta(days=raw_days)).replace(minute=0, second=0, microsecond=0)
+    series: dict[str, list] = {m: [] for m in HISTORY_METRICS}
+
+    # ── Older than the raw window: the hourly rollup ──
+    if since < raw_cutoff:
+        hourly = (await session.execute(
+            select(PluginMetricHourly.metric_name, PluginMetricHourly.bucket,
+                   PluginMetricHourly.value_avg)
+            .where(
+                PluginMetricHourly.source_module == SOURCE_MODULE,
+                PluginMetricHourly.resource_name == host_name,
+                PluginMetricHourly.metric_name.in_(HISTORY_METRICS),
+                PluginMetricHourly.bucket >= since,
+                PluginMetricHourly.bucket < raw_cutoff,
+            )
+            .order_by(PluginMetricHourly.bucket)
+        )).all()
+        for metric_name, bucket, avg in hourly:
+            series[metric_name].append([int(bucket.timestamp() * 1000), round(float(avg), 2)])
+
+    # ── Recent part: raw, bucket-averaged in SQL ──
+    raw_since = since if since >= raw_cutoff else raw_cutoff
+    width_s = bucket_seconds(raw_since, 120)
+    if since < raw_cutoff:
+        width_s = min(width_s, 3600)  # ≤ 1h so it matches the hourly portion
+    rbucket = func.date_bin(
+        func.make_interval(0, 0, 0, 0, 0, 0, width_s),
+        PluginMetric.recorded_at,
+        func.to_timestamp(0),                            # epoch origin
+    ).label("bucket")
+    raw = (await session.execute(
+        select(PluginMetric.metric_name, rbucket, func.avg(PluginMetric.value))
+        .where(
+            PluginMetric.source_module == SOURCE_MODULE,
+            PluginMetric.resource_name == host_name,
+            PluginMetric.metric_name.in_(HISTORY_METRICS),
+            PluginMetric.recorded_at >= raw_since,
+        )
+        .group_by(PluginMetric.metric_name, rbucket)
+        .order_by(rbucket)
+    )).all()
+    for metric_name, b, avg in raw:
+        series[metric_name].append([int(b.timestamp() * 1000), round(float(avg), 2)])
+    return series

plugins/host_agent/plugin.yaml

@@ -1,7 +1,7 @@
 # plugins/host_agent/plugin.yaml
 name: host_agent
-version: "1.1.0"
-description: "Remote Linux host resource monitoring via a lightweight Python push agent (CPU, memory, storage, load, uptime)"
+version: "1.2.0"
+description: "Remote Linux host resource monitoring via a lightweight Python push agent (CPU incl. per-core, memory + PSI, storage, disk I/O, network throughput, load, temperatures, uptime)"
 author: "Steward"
 license: "MIT"
 min_app_version: "0.1.0"

plugins/host_agent/routes.py

@@ -8,21 +8,27 @@ from typing import Any
 from pathlib import Path
 
 import secrets
-from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for
+from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for, session
 from steward.auth.middleware import require_role
 from steward.models.users import UserRole
-from sqlalchemy import select, func
+from sqlalchemy import select, func, or_, and_
 from datetime import timedelta
 
-from steward.core.settings import public_base_url
+from steward.core.settings import get_setting, public_base_url
+from steward.core.time_range import parse_range, RANGE_OPTIONS
 from steward.models.hosts import Host
 from steward.models.metrics import PluginMetric
 from .models import HostAgentRegistration
+# Query helpers live in a model-free module so integration tests can import them
+# without the plugin-loader double-registration guard tripping (see metrics_query).
+from .metrics_query import (
+    SOURCE_MODULE,
+    _history_for_host,
+    _latest_metrics_for_host,
+)
 
 host_agent_bp = Blueprint("host_agent", __name__, template_folder="templates")
 
-SOURCE_MODULE = "host_agent"
-
 
 def _hash_token(raw: str) -> str:
     return hashlib.sha256(raw.encode("utf-8")).hexdigest()
@@ -95,6 +101,60 @@ def _expand_sample_to_metrics(
     if sample.get("storage"):
         row("disk_used_pct_worst", host_name, worst_pct)
 
+    # Per-core CPU. Sub-resources carry a ':' so the fleet widget filters them out.
+    for i, core_pct in enumerate(sample.get("cpu_cores") or []):
+        if core_pct is not None:
+            row("cpu_pct", f"{host_name}:core{i}", core_pct)
+
+    # Richer memory breakdown.
+    if mem.get("cached_bytes") is not None:
+        row("mem_cached_bytes", host_name, mem["cached_bytes"])
+    if mem.get("buffers_bytes") is not None:
+        row("mem_buffers_bytes", host_name, mem["buffers_bytes"])
+
+    # Network throughput — per interface plus a host-level total for the fleet view.
+    net_rx_total = net_tx_total = 0.0
+    for iface in sample.get("net") or []:
+        rx, tx = iface.get("rx_bps", 0.0), iface.get("tx_bps", 0.0)
+        res = f"{host_name}:net:{iface['iface']}"
+        row("net_rx_bps", res, rx)
+        row("net_tx_bps", res, tx)
+        net_rx_total += rx
+        net_tx_total += tx
+    if sample.get("net"):
+        row("net_rx_bps", host_name, net_rx_total)
+        row("net_tx_bps", host_name, net_tx_total)
+
+    # Disk I/O — per device plus host-level total.
+    dr_total = dw_total = 0.0
+    for dev in sample.get("diskio") or []:
+        rd, wr = dev.get("read_bps", 0.0), dev.get("write_bps", 0.0)
+        res = f"{host_name}:diskio:{dev['device']}"
+        row("disk_read_bps", res, rd)
+        row("disk_write_bps", res, wr)
+        dr_total += rd
+        dw_total += wr
+    if sample.get("diskio"):
+        row("disk_read_bps", host_name, dr_total)
+        row("disk_write_bps", host_name, dw_total)
+
+    # Temperatures — per sensor plus the host max for at-a-glance.
+    max_temp: float | None = None
+    for t in sample.get("temps") or []:
+        c = t.get("celsius")
+        if c is None:
+            continue
+        row("temp_c", f"{host_name}:temp:{t['label']}", c)
+        if max_temp is None or c > max_temp:
+            max_temp = c
+    if max_temp is not None:
+        row("temp_c_max", host_name, max_temp)
+
+    # Pressure stall information (PSI) — host-level gauges (mem/cpu/io some|full).
+    for k, v in (sample.get("psi") or {}).items():
+        if isinstance(v, (int, float)):
+            row(f"psi_{k}", host_name, v)
+
     return rows
 
 
@@ -141,6 +201,12 @@ async def ingest():
 
         accepted = 0
         latest_ts: datetime | None = None
+        docker_snapshots: list[tuple[datetime, list]] = []
+        docker_log_batches: list[tuple[datetime, list]] = []
+        latest_swarm: dict | None = None
+        latest_swarm_ts: datetime | None = None
+        latest_disk: dict | None = None
+        latest_disk_ts: datetime | None = None
         for sample in samples:
             try:
                 recorded_at = _parse_ts(sample["ts"])
@@ -149,6 +215,25 @@ async def ingest():
             metrics = _expand_sample_to_metrics(sample, host.name, recorded_at)
             for m in metrics:
                 session.add(m)
+            docker = sample.get("docker")
+            if isinstance(docker, list) and docker:
+                docker_snapshots.append((recorded_at, docker))
+            # Container logs are time-series (append every line, not newest-only);
+            # each record carries its own Docker ts, recorded_at is the fallback.
+            docker_logs = sample.get("docker_logs")
+            if isinstance(docker_logs, list) and docker_logs:
+                docker_log_batches.append((recorded_at, docker_logs))
+            # Swarm is current-state, not time-series — keep only the newest
+            # sample's topology (a manager re-reports it every interval).
+            swarm = sample.get("swarm")
+            if isinstance(swarm, dict) and (
+                    latest_swarm_ts is None or recorded_at > latest_swarm_ts):
+                latest_swarm, latest_swarm_ts = swarm, recorded_at
+            # Disk usage is current-state too — keep only the newest sample's.
+            disk = sample.get("docker_disk")
+            if isinstance(disk, dict) and (
+                    latest_disk_ts is None or recorded_at > latest_disk_ts):
+                latest_disk, latest_disk_ts = disk, recorded_at
             accepted += 1
             if latest_ts is None or recorded_at > latest_ts:
                 latest_ts = recorded_at
@@ -157,6 +242,28 @@ async def ingest():
             await session.rollback()
             return _error(400, "malformed_payload", "no valid samples")
 
+        # Hand host-scoped container data to the docker plugin if it's enabled
+        # (opportunistic synergy via the capability registry — no hard import,
+        # no-op when docker is disabled). A failure here must never sink the
+        # whole ingest, so the metrics above still land.
+        if (docker_snapshots or latest_swarm is not None
+                or latest_disk is not None or docker_log_batches):
+            from steward.core.capabilities import has_capability, invoke_capability
+            if has_capability("docker.persist_host_samples"):
+                try:
+                    # SAVEPOINT so a docker-side failure rolls back only the
+                    # docker writes, leaving the host metrics intact to commit.
+                    async with session.begin_nested():
+                        await invoke_capability(
+                            "docker.persist_host_samples", UserRole.admin,
+                            session, host, docker_snapshots, latest_swarm,
+                            latest_disk, docker_log_batches,
+                        )
+                except Exception:
+                    current_app.logger.exception(
+                        "host_agent ingest: docker persist failed for host=%s",
+                        host.name)
+
         md = payload.get("metadata") or {}
         changed = False
         if latest_ts and (reg.last_seen_at is None or latest_ts > reg.last_seen_at):
@@ -245,10 +352,20 @@ async def agent_source():
     return Response(body, mimetype="text/x-python")
 
 
-@host_agent_bp.get("/widget")
-async def widget_table():
-    """Fleet-glance table: one row per monitored host, latest metrics."""
-    async with current_app.db_sessionmaker() as session:
+def _stale_after_seconds() -> int:
+    return int(
+        current_app.config.get("PLUGINS", {})
+        .get(SOURCE_MODULE, {})
+        .get("stale_after_seconds", 180)
+    )
+
+
+async def _fleet_rows(session) -> list[dict]:
+    """One row per registered host with its latest host-level metrics + stale flag.
+
+    Only bare host-level resources are used here (sub-resources carry a ':' and
+    are skipped) so the fleet glance stays one line per host.
+    """
     regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
     host_ids = [r.host_id for r in regs]
     hosts = {
@@ -276,55 +393,260 @@ async def widget_table():
     )).scalars().all()
 
     latest: dict[str, dict[str, float]] = {}
+    root_disk: dict[str, float] = {}  # host_name → root (/) disk %
     for row in latest_rows:
+        if row.resource_name.endswith(":/") and row.metric_name == "disk_used_pct":
+            root_disk[row.resource_name[:-2]] = row.value
         if ":" in row.resource_name:
             continue
         latest.setdefault(row.resource_name, {})[row.metric_name] = row.value
 
+    # Per-host recent series for the inline sparklines (cpu/mem/load host-level,
+    # disk from the root mount), so each metric shows its trend beside the value —
+    # the host-panel at-a-glance, on the fleet widget. 1h window keeps it cheap.
+    host_names = [h.name for h in hosts.values()]
+    spark_series: dict[str, dict[str, list]] = {
+        n: {"cpu": [], "mem": [], "disk": [], "load": []} for n in host_names
+    }
+    if host_names:
+        since = datetime.now(timezone.utc) - timedelta(hours=1)
+        sp_rows = (await session.execute(
+            select(PluginMetric).where(
+                PluginMetric.source_module == SOURCE_MODULE,
+                PluginMetric.recorded_at >= since,
+                or_(
+                    and_(PluginMetric.resource_name.in_(host_names),
+                         PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))),
+                    and_(PluginMetric.resource_name.in_([n + ":/" for n in host_names]),
+                         PluginMetric.metric_name == "disk_used_pct"),
+                ),
+            ).order_by(PluginMetric.recorded_at)
+        )).scalars().all()
+        for r in sp_rows:
+            if r.resource_name.endswith(":/"):
+                name = r.resource_name[:-2]
+                if name in spark_series:
+                    spark_series[name]["disk"].append(r.value)
+            else:
+                key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name)
+                if key and r.resource_name in spark_series:
+                    spark_series[r.resource_name][key].append(r.value)
+
+    from steward.core.status import sparkline_svg
+
+    stale_after = _stale_after_seconds()
+    now = datetime.now(timezone.utc)
     rows = []
     for reg in regs:
         host = hosts.get(reg.host_id)
         if host is None:
             continue
         m = latest.get(host.name, {})
+        ls = reg.last_seen_at
+        stale = ls is None or (now - ls).total_seconds() > stale_after
+        ser = spark_series.get(host.name, {})
+        sparks = {
+            k: (sparkline_svg(v[-60:], width=72, height=16) if len(v) >= 2 else "")
+            for k, v in ser.items()
+        }
         rows.append({
             "host": host,
             "reg": reg,
+            "stale": stale,
             "cpu_pct": m.get("cpu_pct"),
             "mem_used_pct": m.get("mem_used_pct"),
-            "disk_worst": m.get("disk_used_pct_worst"),
+            "disk_worst": m.get("disk_used_pct_worst"),  # health dot only
+            "disk_root": root_disk.get(host.name),       # displayed at-a-glance
             "load_1m": m.get("load_1m"),
+            "temp_max": m.get("temp_c_max"),
+            "net_rx_bps": m.get("net_rx_bps"),
+            "net_tx_bps": m.get("net_tx_bps"),
+            "disk_read_bps": m.get("disk_read_bps"),
+            "disk_write_bps": m.get("disk_write_bps"),
+            "sparks": sparks,
         })
+    rows.sort(key=lambda r: r["host"].name.lower())
+    return rows
 
+
+@host_agent_bp.get("/widget")
+async def widget_table():
+    """Fleet-glance dashboard widget: one row per monitored host, latest metrics."""
+    async with current_app.db_sessionmaker() as session:
+        rows = await _fleet_rows(session)
     return await render_template("widget_table.html", rows=rows)
 
 
 @host_agent_bp.get("/widget/history")
 async def widget_history():
     host_id = request.args.get("host_id", "")
-    hours = int(request.args.get("hours", "6"))
+    try:
+        hours = max(1, min(168, int(request.args.get("hours", "6"))))
+    except ValueError:
+        hours = 6
+    # wid (the dashboard widget row id) keeps the <canvas> id unique when several
+    # history widgets share a page; fall back to the host id.
+    wid = request.args.get("wid", "") or host_id
     cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
 
+    series: dict[str, list[list]] = {"cpu_pct": [], "mem_used_pct": [], "disk_root": []}
+    host = None
+    if host_id:
+        async with current_app.db_sessionmaker() as session:
+            host = (await session.execute(
+                select(Host).where(Host.id == host_id))).scalar_one_or_none()
+            if host is not None:
+                points = (await session.execute(
+                    select(PluginMetric).where(
+                        PluginMetric.source_module == SOURCE_MODULE,
+                        PluginMetric.recorded_at >= cutoff,
+                        or_(
+                            and_(PluginMetric.resource_name == host.name,
+                                 PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct"))),
+                            and_(PluginMetric.resource_name == host.name + ":/",
+                                 PluginMetric.metric_name == "disk_used_pct"),
+                        ),
+                    ).order_by(PluginMetric.recorded_at)
+                )).scalars().all()
+                # Disk = root (/), consistent with the host panel — not "worst".
+                # Epoch-ms x values let the chart use a plain linear axis (no
+                # Chart.js date adapter needed), matching the host-detail charts.
+                for p in points:
+                    key = "disk_root" if p.resource_name != host.name else p.metric_name
+                    series[key].append([int(p.recorded_at.timestamp() * 1000), round(p.value, 2)])
+
+    # Downsample: the agent reports every few seconds, so a multi-hour window is
+    # hundreds–thousands of points — a noisy, unreadable line. Bucket-average to a
+    # readable point count, keeping the shape.
+    series = {k: _downsample(v) for k, v in series.items()}
+
+    # Stable y-axis ceiling. All three series are percentages, so snap the top up
+    # to the next 10% band (floor 20, cap 100). Auto-scaling to the exact peak made
+    # the chart "zoom" jump on every poll as the peak drifted a few %; snapping to
+    # a band keeps the axis steady across refreshes while still filling the panel.
+    peak = max((p[1] for s in series.values() for p in s), default=0.0)
+    y_max = min(100, max(20, ((int(peak) // 10) + 1) * 10))
+    return await render_template(
+        "widget_history.html", host=host, series=series, hours=hours, wid=wid,
+        y_max=y_max,
+    )
+
+
+def _downsample(series: list[list], target: int = 120) -> list[list]:
+    """Bucket-average a time series [[epoch_ms, value], …] down to ~target points.
+
+    Consecutive samples are grouped into equal buckets and averaged (x = bucket
+    midpoint). Cheap, order-preserving, and smooths the high-frequency noise that
+    makes dense agent series hard to read.
+    """
+    n = len(series)
+    if n <= target:
+        return series
+    bucket = (n + target - 1) // target
+    out: list[list] = []
+    for i in range(0, n, bucket):
+        chunk = series[i:i + bucket]
+        mid = chunk[len(chunk) // 2][0]
+        avg = sum(p[1] for p in chunk) / len(chunk)
+        out.append([mid, round(avg, 2)])
+    return out
+
+
+# Full-metrics refresh cadences. The agent pushes every ~30s by default, so
+# polling the current snapshot at 15s keeps it ≤ one cadence stale; the history
+# charts move slowly over a multi-hour/day window, so they refresh less often.
+_DETAIL_POLL_SECONDS = 15
+_CHART_POLL_SECONDS = 60
+
+
+def _split_host_metrics(host_name: str, latest: dict) -> tuple:
+    """Split a host's latest snapshot into
+    (hostlvl, cores, nets, disks_io, temps, mounts) for the current-state fragment."""
+    hostlvl = latest.get(host_name, {})
+    cores: list = []
+    nets: dict = {}
+    disks_io: dict = {}
+    temps: dict = {}
+    mounts: dict = {}
+    plen = len(host_name) + 1
+    for res, metrics in latest.items():
+        if res == host_name:
+            continue
+        suffix = res[plen:]
+        if suffix.startswith("core"):
+            try:
+                idx = int(suffix[4:])
+            except ValueError:
+                idx = 0
+            cores.append((idx, metrics.get("cpu_pct")))
+        elif suffix.startswith("net:"):
+            nets[suffix[len("net:"):]] = metrics
+        elif suffix.startswith("diskio:"):
+            disks_io[suffix[len("diskio:"):]] = metrics
+        elif suffix.startswith("temp:"):
+            temps[suffix[len("temp:"):]] = metrics.get("temp_c")
+        else:
+            mounts[suffix] = metrics
+    cores.sort(key=lambda c: c[0])
+    return hostlvl, cores, nets, disks_io, temps, mounts
+
+
+@host_agent_bp.get("/<host_id>/")
+@require_role(UserRole.viewer)
+async def host_detail(host_id: str):
+    """Full-metrics shell: header + range toggle only. The current-state and
+    history sections stream in via HTMX (host_detail_metrics / host_detail_charts)
+    so the heavy history query never blocks first paint and the data live-updates."""
+    _, range_key = parse_range(request.args.get("range"))
     async with current_app.db_sessionmaker() as session:
         host = (await session.execute(
             select(Host).where(Host.id == host_id))).scalar_one_or_none()
         if host is None:
             return _error(404, "not_found")
-        points = (await session.execute(
-            select(PluginMetric).where(
-                PluginMetric.source_module == SOURCE_MODULE,
-                PluginMetric.resource_name == host.name,
-                PluginMetric.metric_name.in_(
-                    ("cpu_pct", "mem_used_pct", "disk_used_pct_worst")),
-                PluginMetric.recorded_at >= cutoff,
-            ).order_by(PluginMetric.recorded_at)
-        )).scalars().all()
+    return await render_template(
+        "host_detail.html",
+        host=host, current_range=range_key, range_options=RANGE_OPTIONS,
+        poll_seconds=_DETAIL_POLL_SECONDS, chart_poll_seconds=_CHART_POLL_SECONDS,
+    )
 
-    series: dict[str, list[dict]] = {"cpu_pct": [], "mem_used_pct": [], "disk_used_pct_worst": []}
-    for p in points:
-        series[p.metric_name].append({"t": p.recorded_at.isoformat(), "v": p.value})
 
-    return await render_template("widget_history.html", host=host, series=series, hours=hours)
+@host_agent_bp.get("/<host_id>/metrics")
+@require_role(UserRole.viewer)
+async def host_detail_metrics(host_id: str):
+    """Current-state fragment (gauges/cores/filesystems/IO/temps) — polled live so
+    the page shows the latest snapshot the server holds without a full reload."""
+    async with current_app.db_sessionmaker() as session:
+        host = (await session.execute(
+            select(Host).where(Host.id == host_id))).scalar_one_or_none()
+        if host is None:
+            return "", 404
+        reg = (await session.execute(select(HostAgentRegistration).where(
+            HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
+        latest = await _latest_metrics_for_host(session, host.name)
+    hostlvl, cores, nets, disks_io, temps, mounts = _split_host_metrics(host.name, latest)
+    ls = reg.last_seen_at if reg else None
+    stale = ls is None or (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
+    return await render_template(
+        "_host_metrics.html",
+        host=host, reg=reg, stale=stale, hostlvl=hostlvl,
+        cores=cores, nets=nets, disks_io=disks_io, temps=temps, mounts=mounts,
+    )
+
+
+@host_agent_bp.get("/<host_id>/charts")
+@require_role(UserRole.viewer)
+async def host_detail_charts(host_id: str):
+    """History-charts fragment — lazy-loaded so the date_bin query never blocks
+    the page shell; refreshed on a slow cadence and on range change."""
+    since, range_key = parse_range(request.args.get("range"))
+    async with current_app.db_sessionmaker() as session:
+        host = (await session.execute(
+            select(Host).where(Host.id == host_id))).scalar_one_or_none()
+        if host is None:
+            return "", 404
+        raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7)
+        series = await _history_for_host(session, host.name, since, raw_days=raw_days)
+    return await render_template("_host_charts.html", series=series, range_key=range_key)
 
 
 def _new_token_pair() -> tuple[str, str]:
@@ -332,15 +654,133 @@ def _new_token_pair() -> tuple[str, str]:
     return raw, _hash_token(raw)
 
 
-@host_agent_bp.get("/settings/")
+@host_agent_bp.get("/vitals/<host_id>")
+@require_role(UserRole.viewer)
+async def host_vitals(host_id: str):
+    """Compact live vitals strip (CPU/MEM/DISK/LOAD + pressure) for the Hosts hub.
+
+    Self-contained fragment, polled by hosts/detail.html. Renders nothing until
+    the agent reports — the Agent panel below then carries provisioning.
+    """
+    async with current_app.db_sessionmaker() as db:
+        host = (await db.execute(
+            select(Host).where(Host.id == host_id))).scalar_one_or_none()
+        if host is None:
+            return "", 404
+        reg = (await db.execute(select(HostAgentRegistration).where(
+            HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
+        latest = await _latest_metrics_for_host(db, host.name) if reg else {}
+
+        # Recent series for the at-a-glance sparklines (cpu/mem/load host-level,
+        # disk from the root mount sub-resource).
+        series_raw: dict[str, list[float]] = {"cpu": [], "mem": [], "disk": [], "load": []}
+        if reg:
+            since = datetime.now(timezone.utc) - timedelta(hours=6)
+            spark_rows = (await db.execute(
+                select(PluginMetric).where(
+                    PluginMetric.source_module == SOURCE_MODULE,
+                    PluginMetric.recorded_at >= since,
+                    or_(
+                        and_(PluginMetric.resource_name == host.name,
+                             PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))),
+                        and_(PluginMetric.resource_name == host.name + ":/",
+                             PluginMetric.metric_name == "disk_used_pct"),
+                    ),
+                ).order_by(PluginMetric.recorded_at)
+            )).scalars().all()
+            for r in spark_rows:
+                if r.resource_name == host.name:
+                    key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name)
+                    if key:
+                        series_raw[key].append(r.value)
+                else:
+                    series_raw["disk"].append(r.value)
+
+    from steward.core.status import sparkline_svg
+    hostlvl = latest.get(host.name, {})
+    # At-a-glance disk = the root filesystem (what people actually care about),
+    # not the "worst" mount which is opaque. Worst stays on the full-metrics page.
+    disk_root = (latest.get(host.name + ":/", {}) or {}).get("disk_used_pct")
+    sparks = {k: sparkline_svg(v[-120:]) for k, v in series_raw.items()}
+    # Normalize load by core count so it's comparable across hardware (load/core
+    # as %). Cores derived from the per-core CPU sub-resources already collected.
+    cores = sum(1 for k in latest if k.startswith(host.name + ":core"))
+    load1 = hostlvl.get("load_1m")
+    load_per_core = round(load1 / cores * 100) if (cores and load1 is not None) else None
+    psi = {
+        "cpu": hostlvl.get("psi_cpu_some_avg10"),
+        "mem": hostlvl.get("psi_mem_some_avg10"),
+        "io": hostlvl.get("psi_io_some_avg10"),
+    }
+    ls = reg.last_seen_at if reg else None
+    reporting = bool(reg) and ls is not None
+    stale = reporting and (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
+    return await render_template(
+        "_host_vitals.html",
+        reg=reg, cpu=hostlvl.get("cpu_pct"), mem=hostlvl.get("mem_used_pct"),
+        disk_root=disk_root, load_per_core=load_per_core, sparks=sparks, psi=psi,
+        reporting=reporting, stale=stale,
+    )
+
+
+@host_agent_bp.get("/panel/<host_id>")
+@require_role(UserRole.viewer)
+async def host_panel(host_id: str):
+    """Per-host agent management panel embedded into the Hosts hub via HTMX.
+
+    Lifecycle actions (update / rotate / remove / re-provision) when the host
+    reports, otherwise provisioning — tied to the host's linked Ansible target.
+    The live vitals are a separate strip (host_vitals); this panel is management
+    only. Self-contained fragment (no base.html) so it can be hx-swapped in.
+    """
+    from steward.core.capabilities import has_capability
+    from steward.models.ansible_inventory import AnsibleTarget
+
+    async with current_app.db_sessionmaker() as db:
+        host = (await db.execute(
+            select(Host).where(Host.id == host_id))).scalar_one_or_none()
+        if host is None:
+            return "", 404
+        reg = (await db.execute(select(HostAgentRegistration).where(
+            HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
+        target = (await db.execute(select(AnsibleTarget).where(
+            AnsibleTarget.host_id == host_id))).scalar_one_or_none()
+
+    ls = reg.last_seen_at if reg else None
+    # "Deployed" is gated on the agent actually checking in (last_seen_at), NOT on
+    # the registration row — that row is minted before the playbook runs, so a
+    # failed provision would otherwise look deployed.
+    reporting = bool(reg) and ls is not None
+    stale = reporting and (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
+
+    ansible_cfg = current_app.config.get("ANSIBLE", {})
+    return await render_template(
+        "panel.html",
+        host=host, reg=reg, reporting=reporting, stale=stale, target=target,
+        ansible_available=has_capability("ansible.run_playbook"),
+        managed_key_set=bool((ansible_cfg.get("ssh_public_key") or "").strip()),
+    )
+
+
+@host_agent_bp.get("/fleet/")
 @require_role(UserRole.admin)
-async def settings_list():
-    async with current_app.db_sessionmaker() as session:
-        regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
+async def fleet():
+    from steward.core.capabilities import has_capability
+    from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup
+
+    ansible_available = has_capability("ansible.run_playbook")
+    async with current_app.db_sessionmaker() as db:
+        regs = (await db.execute(select(HostAgentRegistration))).scalars().all()
         hosts_by_id = {
-            h.id: h for h in (await session.execute(
+            h.id: h for h in (await db.execute(
                 select(Host).where(Host.id.in_([r.host_id for r in regs])))).scalars().all()
         } if regs else {}
+        targets, groups = [], []
+        if ansible_available:
+            targets = (await db.execute(
+                select(AnsibleTarget).order_by(AnsibleTarget.name))).scalars().all()
+            groups = (await db.execute(
+                select(AnsibleGroup).order_by(AnsibleGroup.name))).scalars().all()
 
     new_token = request.args.get("new_token")
     new_host_id = request.args.get("host_id")
@@ -348,6 +788,9 @@ async def settings_list():
     if new_token and new_host_id:
         install_url = f"{public_base_url(request)}/plugins/host_agent/install.sh?token={new_token}"
 
+    managed_key_set = bool(
+        (current_app.config.get("ANSIBLE", {}).get("ssh_public_key") or "").strip())
+
     return await render_template(
         "settings_list.html",
         registrations=[
@@ -355,10 +798,14 @@ async def settings_list():
         ],
         new_token=new_token,
         install_url=install_url,
+        ansible_available=ansible_available,
+        deploy_targets=targets,
+        deploy_groups=groups,
+        managed_key_set=managed_key_set,
     )
 
 
-@host_agent_bp.post("/settings/add-host")
+@host_agent_bp.post("/fleet/add-host")
 @require_role(UserRole.admin)
 async def add_host():
     form = await request.form
@@ -387,10 +834,10 @@ async def add_host():
         await session.commit()
         host_id = host.id
 
-    return redirect(url_for("host_agent.settings_list") + f"?new_token={raw}&host_id={host_id}")
+    return redirect(url_for("host_agent.fleet") + f"?new_token={raw}&host_id={host_id}")
 
 
-@host_agent_bp.post("/settings/<host_id>/rotate-token")
+@host_agent_bp.post("/fleet/<host_id>/rotate-token")
 @require_role(UserRole.admin)
 async def rotate_token(host_id: str):
     async with current_app.db_sessionmaker() as session:
@@ -403,10 +850,10 @@ async def rotate_token(host_id: str):
         reg.token_hash = hashed
         reg.token_created_at = datetime.now(timezone.utc)
         await session.commit()
-    return redirect(url_for("host_agent.settings_list") + f"?new_token={raw}&host_id={host_id}")
+    return redirect(url_for("host_agent.fleet") + f"?new_token={raw}&host_id={host_id}")
 
 
-@host_agent_bp.post("/settings/<host_id>/delete")
+@host_agent_bp.post("/fleet/<host_id>/delete")
 @require_role(UserRole.admin)
 async def delete_registration(host_id: str):
     async with current_app.db_sessionmaker() as session:
@@ -416,4 +863,220 @@ async def delete_registration(host_id: str):
         if reg is not None:
             await session.delete(reg)
             await session.commit()
-    return redirect(url_for("host_agent.settings_list"))
+    return redirect(url_for("host_agent.fleet"))
+
+
+# ── Deploy via Ansible (plugin↔core synergy via the capability registry) ──────
+
+async def _ensure_host_for_target(db, target) -> "Host":
+    """Find or create the Host that an AnsibleTarget should report under."""
+    if getattr(target, "host_id", None):
+        h = await db.get(Host, target.host_id)
+        if h:
+            return h
+    h = (await db.execute(
+        select(Host).where(Host.name == target.name))).scalar_one_or_none()
+    if h is None:
+        h = Host(name=target.name, address=getattr(target, "address", "") or "")
+        db.add(h)
+        await db.flush()
+    return h
+
+
+async def _mint_registration_token(db, host) -> str:
+    """Create or rotate the host's agent registration; return the raw token.
+
+    Tokens are stored hashed (unrecoverable), so deploying always mints a fresh
+    token — the run installs the agent with it. Existing agents get rotated.
+    """
+    reg = (await db.execute(select(HostAgentRegistration).where(
+        HostAgentRegistration.host_id == host.id))).scalar_one_or_none()
+    raw, hashed = _new_token_pair()
+    if reg is None:
+        db.add(HostAgentRegistration(host_id=host.id, token_hash=hashed))
+    else:
+        reg.token_hash = hashed
+        reg.token_created_at = datetime.now(timezone.utc)
+    return raw
+
+
+@host_agent_bp.post("/deploy")
+@require_role(UserRole.admin)
+async def deploy_via_ansible():
+    """Install/update the agent on Ansible inventory targets via the bundled
+    install playbook — the 'curl | sh becomes a button' synergy.
+
+    Uses the core "ansible.run_playbook" capability (no hard import of the
+    runner) and injects a freshly-minted per-host token as an inventory hostvar.
+    """
+    from steward.core.capabilities import has_capability, invoke_capability
+    from steward.ansible.inventory_gen import (
+        fetch_scope_targets, generate_inventory, inventory_to_yaml)
+    from steward.ansible.sources import BUILTIN_SOURCE_NAME
+
+    if not has_capability("ansible.run_playbook"):
+        return _error(400, "ansible_unavailable", "Ansible is not available")
+
+    form = await request.form
+    scope = (form.get("inventory_scope", "") or "").strip()
+    try:
+        interval = max(5, int(form.get("agent_interval", "30")))
+    except (TypeError, ValueError):
+        interval = 30
+    if not (scope.startswith("steward:target:")
+            or scope.startswith("steward:group:")
+            or scope == "steward:all"):
+        return _error(400, "bad_scope", "Choose a target or group")
+
+    url = public_base_url(request)
+    async with current_app.db_sessionmaker() as db:
+        targets = await fetch_scope_targets(db, scope)
+        if not targets:
+            return _error(400, "no_targets", "No Ansible targets in that scope")
+        # The read above autobegins the session transaction (SQLAlchemy 2.0), so
+        # mint into it directly and commit — a nested db.begin() would double-begin.
+        tokens: dict[str, str] = {}
+        for t in targets:
+            host = await _ensure_host_for_target(db, t)
+            tokens[t.name] = await _mint_registration_token(db, host)
+        inv = generate_inventory(targets)  # built while ORM objects are still live
+        # Per-host token as a host var; steward_url goes in as an extra-var below.
+        for name, tok in tokens.items():
+            inv["_meta"]["hostvars"].setdefault(name, {})["steward_token"] = tok
+        inventory_content = inventory_to_yaml(inv)
+        await db.commit()
+
+    actor_role = UserRole(session.get("user_role", "viewer"))
+    run, _source, err = await invoke_capability(
+        "ansible.run_playbook", actor_role,
+        current_app._get_current_object(),  # type: ignore[attr-defined]
+        source_name=BUILTIN_SOURCE_NAME,
+        playbook_path="host_agent/install.yml",
+        inventory_content=inventory_content,
+        inventory_scope=scope,
+        # extra-vars outrank the playbook's `vars:` defaults; token stays per-host.
+        params={"extra_vars_map": {"steward_url": url, "agent_interval": interval}},
+        triggered_by=session.get("user_id"),
+    )
+    if err:
+        return _error(400, "deploy_failed", err)
+    return redirect(url_for("ansible.run_detail", run_id=run.id))
+
+
+@host_agent_bp.post("/provision")
+@require_role(UserRole.admin)
+async def provision_via_ansible():
+    """First-contact provisioning: create the steward account + install the
+    managed key + NOPASSWD sudo, then install the agent — over bootstrap
+    (password) auth. After this, hosts are reachable with the managed key.
+
+    The bootstrap password is passed to the runner as a connection override and
+    is NOT persisted on the run row.
+    """
+    from steward.core.capabilities import has_capability, invoke_capability
+    from steward.ansible.inventory_gen import (
+        fetch_scope_targets, generate_inventory, inventory_to_yaml)
+    from steward.ansible.sources import BUILTIN_SOURCE_NAME
+
+    if not has_capability("ansible.run_playbook"):
+        return _error(400, "ansible_unavailable", "Ansible is not available")
+
+    ansible_cfg = current_app.config.get("ANSIBLE", {})
+    pubkey = (ansible_cfg.get("ssh_public_key") or "").strip()
+    if not pubkey:
+        return _error(400, "no_managed_key",
+                      "Generate a managed SSH key in Settings → Ansible first")
+    steward_user = (ansible_cfg.get("ssh_user") or "steward").strip() or "steward"
+
+    form = await request.form
+    scope = (form.get("inventory_scope", "") or "").strip()
+    boot_user = (form.get("bootstrap_user", "") or "").strip()
+    boot_password = form.get("bootstrap_password", "") or ""
+    try:
+        interval = max(5, int(form.get("agent_interval", "30")))
+    except (TypeError, ValueError):
+        interval = 30
+    if not (scope.startswith("steward:target:")
+            or scope.startswith("steward:group:")
+            or scope == "steward:all"):
+        return _error(400, "bad_scope", "Choose a target or group")
+    if not boot_user or not boot_password:
+        return _error(400, "missing_bootstrap",
+                      "Bootstrap user and password are required for first contact")
+
+    url = public_base_url(request)
+    async with current_app.db_sessionmaker() as db:
+        targets = await fetch_scope_targets(db, scope)
+        if not targets:
+            return _error(400, "no_targets", "No Ansible targets in that scope")
+        # The read above autobegins the session transaction (SQLAlchemy 2.0), so
+        # mint into it directly and commit — a nested db.begin() would double-begin.
+        tokens: dict[str, str] = {}
+        for t in targets:
+            host = await _ensure_host_for_target(db, t)
+            tokens[t.name] = await _mint_registration_token(db, host)
+        inv = generate_inventory(targets)  # built while ORM objects are still live
+        # Only the per-host token is a host var; the globals go in as extra-vars
+        # below (extra-vars outrank play vars, host vars do not).
+        for name, tok in tokens.items():
+            inv["_meta"]["hostvars"].setdefault(name, {})["steward_token"] = tok
+        inventory_content = inventory_to_yaml(inv)
+        await db.commit()
+
+    actor_role = UserRole(session.get("user_role", "viewer"))
+    run, _source, err = await invoke_capability(
+        "ansible.run_playbook", actor_role,
+        current_app._get_current_object(),  # type: ignore[attr-defined]
+        source_name=BUILTIN_SOURCE_NAME,
+        playbook_path="host_agent/provision.yml",
+        inventory_content=inventory_content,
+        inventory_scope=scope,
+        # extra_vars_map → a JSON -e @file: highest precedence (beats play vars)
+        # and space-safe (steward_pubkey carries a comment with a space).
+        params={"extra_vars_map": {
+            "steward_url": url,
+            "steward_pubkey": pubkey,
+            "steward_user": steward_user,
+            "agent_interval": interval,
+        }},
+        triggered_by=session.get("user_id"),
+        connection={"user": boot_user, "password": boot_password},
+    )
+    if err:
+        return _error(400, "provision_failed", err)
+    return redirect(url_for("ansible.run_detail", run_id=run.id))
+
+
+@host_agent_bp.post("/update")
+@require_role(UserRole.admin)
+async def update_via_ansible():
+    """Refresh agent.py on already-installed hosts via the bundled update
+    playbook. Connects as the managed steward account — no token rotation, no
+    config rewrite (the host keeps its existing identity)."""
+    from steward.core.capabilities import has_capability, invoke_capability
+    from steward.ansible.sources import BUILTIN_SOURCE_NAME
+
+    if not has_capability("ansible.run_playbook"):
+        return _error(400, "ansible_unavailable", "Ansible is not available")
+
+    form = await request.form
+    scope = (form.get("inventory_scope", "") or "").strip()
+    if not (scope.startswith("steward:target:")
+            or scope.startswith("steward:group:")
+            or scope == "steward:all"):
+        return _error(400, "bad_scope", "Choose a target or group")
+
+    url = public_base_url(request)
+    actor_role = UserRole(session.get("user_role", "viewer"))
+    run, _source, err = await invoke_capability(
+        "ansible.run_playbook", actor_role,
+        current_app._get_current_object(),  # type: ignore[attr-defined]
+        source_name=BUILTIN_SOURCE_NAME,
+        playbook_path="host_agent/update.yml",
+        inventory_scope=scope,
+        params={"extra_vars": [f"steward_url={url}"]},
+        triggered_by=session.get("user_id"),
+    )
+    if err:
+        return _error(400, "update_failed", err)
+    return redirect(url_for("ansible.run_detail", run_id=run.id))

plugins/host_agent/templates/_host_charts.html

@@ -0,0 +1,6 @@
+{# History data-only fragment. Swapped into the hidden #hm-charts-data poller;
+   the script runs and feeds the persistent charts created by host_detail.html,
+   updating them in place (no canvas swap → no flicker / no re-animation). #}
+<script>
+  if (window.applyHostSeries) window.applyHostSeries({{ series|tojson }}, {{ range_key|tojson }});
+</script>

plugins/host_agent/templates/_host_metrics.html

@@ -0,0 +1,182 @@
+{# Current-state fragment for the full-metrics page — polled live via HTMX.
+   Self-contained (defines its own format macros) so each poll re-renders the
+   latest snapshot the server has. #}
+{% macro fmt_bps(v) %}
+{%- if v is none -%}—
+{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s
+{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s
+{%- else -%}{{ "%.0f"|format(v) }} B/s
+{%- endif -%}
+{% endmacro %}
+{% macro fmt_bytes(v) %}
+{%- if v is none -%}—
+{%- elif v >= 1125899906842624 -%}{{ "%.1f"|format(v / 1125899906842624) }} PB
+{%- elif v >= 1099511627776 -%}{{ "%.1f"|format(v / 1099511627776) }} TB
+{%- elif v >= 1073741824 -%}{{ "%.1f"|format(v / 1073741824) }} GB
+{%- elif v >= 1048576 -%}{{ "%.0f"|format(v / 1048576) }} MB
+{%- else -%}{{ "%.0f"|format(v / 1024) }} KB
+{%- endif -%}
+{% endmacro %}
+{% macro bar(pct) %}
+{%- set p = pct if pct is not none else 0 -%}
+<div style="height:8px;background:var(--bg-elevated);border-radius:4px;overflow:hidden;border:1px solid var(--border);">
+  <div style="height:100%;width:{{ p|round(1) }}%;background:{% if pct is none %}var(--text-dim){% elif pct < 70 %}var(--green){% elif pct < 90 %}var(--yellow){% else %}var(--red){% endif %};"></div>
+</div>
+{% endmacro %}
+
+{# ── Identity / metadata ─────────────────────────────────────────────────── #}
+<div class="card" style="display:flex;flex-wrap:wrap;gap:0.5rem 1.25rem;align-items:center;font-size:0.82rem;color:var(--text-muted);">
+  <span>{% if stale %}<span class="badge badge-red">stale</span>{% else %}<span class="badge badge-green">live</span>{% endif %}</span>
+  <span><span style="color:var(--text-dim);">Address</span> {{ host.address or "—" }}</span>
+  {% if reg %}
+  {% if reg.distro %}<span><span style="color:var(--text-dim);">OS</span> {{ reg.distro }}</span>{% endif %}
+  {% if reg.kernel %}<span><span style="color:var(--text-dim);">Kernel</span> {{ reg.kernel }}</span>{% endif %}
+  {% if reg.arch %}<span><span style="color:var(--text-dim);">Arch</span> {{ reg.arch }}</span>{% endif %}
+  {% if reg.agent_version %}<span><span style="color:var(--text-dim);">Agent</span> v{{ reg.agent_version }}</span>{% endif %}
+  {% set up = hostlvl.get('uptime_secs') %}
+  {% if up %}<span><span style="color:var(--text-dim);">Uptime</span> {{ (up // 86400)|int }}d {{ ((up % 86400) // 3600)|int }}h</span>{% endif %}
+  <span><span style="color:var(--text-dim);">Last seen</span> {{ reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") ~ " UTC" if reg.last_seen_at else "never" }}</span>
+  {% else %}
+  <span>No agent registration found for this host.</span>
+  {% endif %}
+</div>
+
+{% if not hostlvl %}
+<div class="card empty">No metrics reported yet. Once the agent checks in, CPU, memory, network, disk I/O, temperatures, and pressure will appear here.</div>
+{% else %}
+
+{# ── Current headline gauges ─────────────────────────────────────────────── #}
+<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1rem;">
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.35rem;">CPU</div>
+    {% set c = hostlvl.get('cpu_pct') %}
+    <span class="stat-val" style="font-size:1.6rem;color:{% if c is none %}var(--text-dim){% elif c < 70 %}var(--green){% elif c < 90 %}var(--yellow){% else %}var(--red){% endif %};">{% if c is not none %}{{ "%.0f"|format(c) }}%{% else %}—{% endif %}</span>
+    <div style="margin-top:0.5rem;">{{ bar(c) }}</div>
+  </div>
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.35rem;">Memory</div>
+    {% set mp = hostlvl.get('mem_used_pct') %}
+    <span class="stat-val" style="font-size:1.6rem;color:{% if mp is none %}var(--text-dim){% elif mp < 70 %}var(--green){% elif mp < 90 %}var(--yellow){% else %}var(--red){% endif %};">{% if mp is not none %}{{ "%.0f"|format(mp) }}%{% else %}—{% endif %}</span>
+    <div style="margin-top:0.5rem;">{{ bar(mp) }}</div>
+    <div style="margin-top:0.45rem;font-size:0.72rem;color:var(--text-dim);">
+      avail {{ fmt_bytes(hostlvl.get('mem_available_bytes')) }} · cache {{ fmt_bytes(hostlvl.get('mem_cached_bytes')) }} · swap {{ fmt_bytes(hostlvl.get('swap_used_bytes')) }}
+    </div>
+  </div>
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.35rem;">Load</div>
+    <span class="stat-val" style="font-size:1.6rem;">{% if hostlvl.get('load_1m') is not none %}{{ "%.2f"|format(hostlvl.get('load_1m')) }}{% else %}—{% endif %}</span>
+    <div style="margin-top:0.45rem;font-size:0.72rem;color:var(--text-dim);">
+      5m {% if hostlvl.get('load_5m') is not none %}{{ "%.2f"|format(hostlvl.get('load_5m')) }}{% else %}—{% endif %} ·
+      15m {% if hostlvl.get('load_15m') is not none %}{{ "%.2f"|format(hostlvl.get('load_15m')) }}{% else %}—{% endif %}
+    </div>
+  </div>
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.35rem;">Network</div>
+    <div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
+      <div><span style="color:var(--green);">↓</span> {{ fmt_bps(hostlvl.get('net_rx_bps')) }}</div>
+      <div><span style="color:var(--red);">↑</span> {{ fmt_bps(hostlvl.get('net_tx_bps')) }}</div>
+    </div>
+  </div>
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.35rem;">Disk I/O</div>
+    <div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
+      <div><span style="color:var(--text-dim);">rd</span> {{ fmt_bps(hostlvl.get('disk_read_bps')) }}</div>
+      <div><span style="color:var(--text-dim);">wr</span> {{ fmt_bps(hostlvl.get('disk_write_bps')) }}</div>
+    </div>
+  </div>
+  {% if hostlvl.get('temp_c_max') is not none %}
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.35rem;">Temp (max)</div>
+    {% set t = hostlvl.get('temp_c_max') %}
+    <span class="stat-val" style="font-size:1.6rem;color:{% if t < 70 %}var(--green){% elif t < 85 %}var(--yellow){% else %}var(--red){% endif %};">{{ "%.0f"|format(t) }}°C</span>
+  </div>
+  {% endif %}
+  {% if hostlvl.get('psi_mem_some_avg10') is not none %}
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.35rem;" title="Pressure stall — % of time stalled, 10s avg">Pressure (10s)</div>
+    <div style="font-size:0.78rem;font-variant-numeric:tabular-nums;color:var(--text-muted);">
+      <div>mem {{ "%.1f"|format(hostlvl.get('psi_mem_some_avg10')) }}%</div>
+      {% if hostlvl.get('psi_cpu_some_avg10') is not none %}<div>cpu {{ "%.1f"|format(hostlvl.get('psi_cpu_some_avg10')) }}%</div>{% endif %}
+      {% if hostlvl.get('psi_io_some_avg10') is not none %}<div>io {{ "%.1f"|format(hostlvl.get('psi_io_some_avg10')) }}%</div>{% endif %}
+    </div>
+  </div>
+  {% endif %}
+</div>
+
+{# ── Per-core CPU ─────────────────────────────────────────────────────────── #}
+{% if cores %}
+<div class="card">
+  <div class="section-title" style="margin-bottom:0.6rem;">Per-core CPU</div>
+  <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(120px,1fr));gap:0.5rem 0.9rem;">
+    {% for idx, pct in cores %}
+    <div>
+      <div style="display:flex;justify-content:space-between;font-size:0.72rem;color:var(--text-dim);margin-bottom:0.2rem;">
+        <span>core {{ idx }}</span><span>{% if pct is not none %}{{ "%.0f"|format(pct) }}%{% else %}—{% endif %}</span>
+      </div>
+      {{ bar(pct) }}
+    </div>
+    {% endfor %}
+  </div>
+</div>
+{% endif %}
+
+{# ── Filesystems ──────────────────────────────────────────────────────────── #}
+{% if mounts %}
+<div class="card">
+  <div class="section-title" style="margin-bottom:0.6rem;">Filesystems</div>
+  {% for mount, m in mounts.items() %}
+  <div style="margin-bottom:0.6rem;">
+    <div style="display:flex;justify-content:space-between;font-size:0.8rem;margin-bottom:0.2rem;">
+      <span style="font-family:ui-monospace,monospace;">{{ mount }}</span>
+      <span style="color:var(--text-muted);">{{ fmt_bytes(m.get('disk_used_bytes')) }} / {{ fmt_bytes(m.get('disk_total_bytes')) }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %}</span>
+    </div>
+    {{ bar(m.get('disk_used_pct')) }}
+  </div>
+  {% endfor %}
+</div>
+{% endif %}
+
+{# ── Interfaces / disks (short panels, side by side at natural height) ─────── #}
+{% if nets or disks_io %}
+<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(260px,1fr));gap:1rem;align-items:start;">
+  {% if nets %}
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.5rem;">Interfaces</div>
+    {% for iface, m in nets.items() %}
+    <div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
+      <span style="font-family:ui-monospace,monospace;">{{ iface }}</span>
+      <span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">↓ {{ fmt_bps(m.get('net_rx_bps')) }} · ↑ {{ fmt_bps(m.get('net_tx_bps')) }}</span>
+    </div>
+    {% endfor %}
+  </div>
+  {% endif %}
+  {% if disks_io %}
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.5rem;">Disks</div>
+    {% for dev, m in disks_io.items() %}
+    <div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
+      <span style="font-family:ui-monospace,monospace;">{{ dev }}</span>
+      <span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">rd {{ fmt_bps(m.get('disk_read_bps')) }} · wr {{ fmt_bps(m.get('disk_write_bps')) }}</span>
+    </div>
+    {% endfor %}
+  </div>
+  {% endif %}
+</div>
+{% endif %}
+
+{# ── Temperatures (full width; cores flow into a compact multi-column grid) ── #}
+{% if temps %}
+<div class="card" style="margin-top:1rem;margin-bottom:0;">
+  <div class="section-title" style="margin-bottom:0.5rem;">Temperatures</div>
+  <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(130px,1fr));gap:0.1rem 1.25rem;">
+    {% for label, c in temps.items() %}
+    <div style="display:flex;justify-content:space-between;gap:0.5rem;font-size:0.8rem;padding:0.15rem 0;">
+      <span style="white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ label }}</span>
+      <span style="color:{% if c is none %}var(--text-dim){% elif c < 70 %}var(--green){% elif c < 85 %}var(--yellow){% else %}var(--red){% endif %};font-variant-numeric:tabular-nums;">{% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %}</span>
+    </div>
+    {% endfor %}
+  </div>
+</div>
+{% endif %}
+
+{% endif %}

plugins/host_agent/templates/_host_vitals.html

@@ -0,0 +1,35 @@
+{# Host vitals strip — compact CPU/MEM/DISK/LOAD + pressure, live-polled into the
+   host hub. Renders nothing until the agent reports (the Agent panel below then
+   shows provisioning). #}
+{% if reporting %}
+{% set lbl = "font-size:0.68rem;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em;" %}
+{% macro vital(label, value, text, kind, spark) %}
+<div style="min-width:88px;">
+  <div style="{{ lbl }}">{{ label }}</div>
+  <div style="font-size:1.5rem;font-weight:700;line-height:1.1;{{ threshold_style(value, kind) }}">{{ text }}</div>
+  <div style="margin-top:0.2rem;">{{ spark | safe }}</div>
+</div>
+{% endmacro %}
+<div class="card" style="display:flex;flex-wrap:wrap;align-items:flex-start;gap:1rem 2rem;margin-bottom:1rem;">
+  {{ vital("CPU", cpu, ('%.0f%%'|format(cpu) if cpu is not none else '—'), 'cpu', sparks.cpu) }}
+  {{ vital("Memory", mem, ('%.0f%%'|format(mem) if mem is not none else '—'), 'mem', sparks.mem) }}
+  {{ vital("Disk /", disk_root, ('%.0f%%'|format(disk_root) if disk_root is not none else '—'), 'disk', sparks.disk) }}
+  {{ vital("Load /core", load_per_core, ('%d%%'|format(load_per_core) if load_per_core is not none else '—'), 'load', sparks.load) }}
+
+  <div style="margin-left:auto;display:flex;flex-direction:column;align-items:flex-end;gap:0.3rem;font-size:0.75rem;color:var(--text-muted);text-align:right;">
+    <span>
+      {% if stale %}<span class="badge badge-red">stale</span>{% else %}<span class="badge badge-green">live</span>{% endif %}
+      {% if reg and reg.agent_version %}<span style="margin-left:0.3rem;">v{{ reg.agent_version }}</span>{% endif %}
+    </span>
+    {% if psi.cpu is not none or psi.mem is not none or psi.io is not none %}
+    <span title="Pressure stall — % of the last 10s tasks waited for the resource">
+      <span style="color:var(--text-dim);text-transform:uppercase;font-size:0.66rem;letter-spacing:0.04em;">Pressure 10s</span>
+      cpu {{ '%.0f%%'|format(psi.cpu) if psi.cpu is not none else '—' }}
+      · mem {{ '%.0f%%'|format(psi.mem) if psi.mem is not none else '—' }}
+      · io {{ '%.0f%%'|format(psi.io) if psi.io is not none else '—' }}
+    </span>
+    {% endif %}
+    {% if reg and reg.last_seen_at %}<span>seen {{ reg.last_seen_at.strftime("%H:%M:%S") }} UTC</span>{% endif %}
+  </div>
+</div>
+{% endif %}

plugins/host_agent/templates/host_detail.html

@@ -0,0 +1,108 @@
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs %}
+{% block title %}{{ host.name }} — Metrics — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), (host.name, "/hosts/" ~ host.id), ("Metrics", "")]) }}{% endblock %}
+{% block content %}
+{# Shell only: current-state + history data stream in via HTMX so the heavy
+   history query never blocks first paint and the data refreshes live (≈ the
+   agent's push cadence). The chart CANVASES live here permanently — only their
+   data is polled and applied in place, so refreshes never re-create/flicker. #}
+<div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;flex-wrap:wrap;margin-bottom:0.75rem;">
+  <h1 class="page-title" style="margin-bottom:0;">{{ host.name }}</h1>
+  {% include "_time_range.html" %}
+</div>
+
+{# Current state — lazy + polled live. Atomic innerHTML swap of fixed-height
+   cards, so values update without a layout jump. #}
+<div id="hm-current"
+     hx-get="/plugins/host_agent/{{ host.id }}/metrics"
+     hx-trigger="load, every {{ poll_seconds }}s"
+     hx-swap="innerHTML">
+  <div class="card empty">Loading metrics…</div>
+</div>
+
+{# History charts — persistent canvases (created once below); only data is polled. #}
+<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(340px,1fr));gap:1rem;margin-top:1rem;">
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.5rem;">Utilization % — last <span class="hm-chart-range">{{ current_range }}</span></div>
+    <div style="height:220px;"><canvas id="chart-util"></canvas></div>
+  </div>
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.5rem;">Throughput (B/s) — last <span class="hm-chart-range">{{ current_range }}</span></div>
+    <div style="height:220px;"><canvas id="chart-net"></canvas></div>
+  </div>
+  <div class="card" style="margin-bottom:0;">
+    <div class="section-title" style="margin-bottom:0.5rem;">Load &amp; pressure — last <span class="hm-chart-range">{{ current_range }}</span></div>
+    <div style="height:220px;"><canvas id="chart-pressure"></canvas></div>
+  </div>
+</div>
+{# Hidden poller: fetches the history series and applies it to the charts above
+   (no visible swap). Lazy on load → never blocks paint; refreshed slowly + on
+   range change. #}
+<div id="hm-charts-data" style="display:none;"
+     hx-get="/plugins/host_agent/{{ host.id }}/charts"
+     hx-trigger="load, every {{ chart_poll_seconds }}s, rangeChange from:body"
+     hx-include="#time-range"
+     hx-swap="innerHTML"></div>
+
+<script>
+(function () {
+  const fmtTime = (v) => { const d = new Date(v); return ("0" + d.getHours()).slice(-2) + ":" + ("0" + d.getMinutes()).slice(-2); };
+  const baseOpts = (ymax) => ({
+    responsive: true, maintainAspectRatio: false,
+    animation: false,                       // no grow-in / re-animate on refresh
+    interaction: { mode: "index", intersect: false },
+    elements: { point: { radius: 0 } },
+    scales: {
+      x: { type: "linear", ticks: { callback: (v) => fmtTime(v), maxTicksLimit: 8, color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
+      y: { beginAtZero: true, max: ymax, ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
+    },
+    plugins: {
+      legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } },
+      tooltip: { callbacks: { title: (items) => items.length ? fmtTime(items[0].parsed.x) : "" } },
+    },
+  });
+  const defs = {
+    "chart-util": { ymax: 100, ds: [
+      { key: "cpu_pct", label: "CPU %", color: "#c8a840" },
+      { key: "mem_used_pct", label: "Mem %", color: "#4aa86a" },
+      { key: "disk_used_pct_worst", label: "Disk % (worst)", color: "#c87840" },
+    ] },
+    "chart-net": { ymax: undefined, ds: [
+      { key: "net_rx_bps", label: "Net RX", color: "#4aa86a" },
+      { key: "net_tx_bps", label: "Net TX", color: "#c84048" },
+      { key: "disk_read_bps", label: "Disk read", color: "#6aa0d0" },
+      { key: "disk_write_bps", label: "Disk write", color: "#c8a840" },
+    ] },
+    "chart-pressure": { ymax: undefined, ds: [
+      { key: "load_1m", label: "Load 1m", color: "#c8a840" },
+      { key: "psi_mem_some_avg10", label: "Mem PSI %", color: "#c84048" },
+      { key: "temp_c_max", label: "Temp °C", color: "#c87840" },
+    ] },
+  };
+  const charts = {};
+  for (const id of Object.keys(defs)) {
+    const el = document.getElementById(id);
+    if (!el) continue;
+    charts[id] = new Chart(el.getContext("2d"), {
+      type: "line",
+      data: { datasets: defs[id].ds.map((d) => ({ label: d.label, data: [], borderColor: d.color, backgroundColor: d.color, borderWidth: 1.5, tension: 0.25 })) },
+      options: baseOpts(defs[id].ymax),
+    });
+  }
+  // Called by the polled /charts data fragment. Updates data in place (no
+  // re-create, no animation), so refreshes never flicker or shift layout.
+  window.applyHostSeries = function (series, rangeKey) {
+    for (const id of Object.keys(defs)) {
+      const ch = charts[id];
+      if (!ch) continue;
+      defs[id].ds.forEach((d, i) => {
+        ch.data.datasets[i].data = (series[d.key] || []).map((p) => ({ x: p[0], y: p[1] }));
+      });
+      ch.update("none");
+    }
+    if (rangeKey) document.querySelectorAll(".hm-chart-range").forEach((s) => { s.textContent = rangeKey; });
+  };
+})();
+</script>
+{% endblock %}

plugins/host_agent/templates/install.sh.j2

@@ -45,6 +45,10 @@ cat > "$CONF_FILE" <<EOF
 url = $STEWARD_URL
 token = $AGENT_TOKEN
 interval_seconds = 30
+# Container logs are collected by default. To opt this host out entirely:
+#   docker_logs_enabled = false
+# To skip specific noisy containers (comma-separated names):
+#   docker_log_exclude = watchtower, some-chatty-service
 EOF
 chown "root:$AGENT_USER" "$CONF_FILE"
 chmod 0640 "$CONF_FILE"

plugins/host_agent/templates/panel.html

@@ -0,0 +1,146 @@
+{# Per-host agent panel — embedded into /hosts/<id> via HTMX. Self-contained. #}
+{% set scope = "steward:target:" ~ target.id if target else "" %}
+<div class="card">
+  <div style="display:flex;align-items:baseline;justify-content:space-between;margin-bottom:0.75rem;gap:1rem;flex-wrap:wrap;">
+    <h3 class="section-title" style="margin-bottom:0;">Agent</h3>
+    {% if reporting %}
+    <span style="font-size:0.78rem;color:{{ 'var(--yellow)' if stale else 'var(--green)' }};">
+      {{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %}
+      · last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }}
+    </span>
+    {% elif reg %}
+    <span style="font-size:0.78rem;color:var(--text-muted);">pending — no check-in yet</span>
+    {% endif %}
+  </div>
+
+  {% if reporting %}
+  {# Reporting: live vitals are shown by the vitals strip above; this panel is
+     lifecycle/management only. #}
+  <div style="display:flex;gap:0.5rem;flex-wrap:wrap;align-items:center;">
+    <a href="/plugins/host_agent/{{ host.id }}/" class="btn btn-sm btn-ghost">Full metrics →</a>
+    {% if session.user_role == 'admin' %}
+    {% if ansible_available and target %}
+    <form method="post" action="/plugins/host_agent/update" style="margin:0;">
+      <input type="hidden" name="inventory_scope" value="{{ scope }}">
+      <button type="submit" class="btn btn-sm" title="Refresh agent.py + restart (token preserved)">Update agent</button>
+    </form>
+    {% endif %}
+    <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/rotate-token" style="margin:0;"
+          onsubmit="return confirm('Rotate token? The agent stops reporting until reinstalled/updated with the new token.');">
+      <button type="submit" class="btn btn-sm btn-ghost">Rotate token</button>
+    </form>
+    <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0;"
+          onsubmit="return confirm('Remove this agent registration? Metrics stop until re-registered.');">
+      <button type="submit" class="btn btn-sm btn-danger">Remove</button>
+    </form>
+    {% endif %}
+  </div>
+  {% if ansible_available and not target %}
+  <p style="color:var(--text-dim);font-size:0.8rem;margin:0.6rem 0 0;">
+    Link an Ansible target (above) to enable one-click agent updates.
+  </p>
+  {% endif %}
+
+  {% if session.user_role == 'admin' and ansible_available and target and managed_key_set %}
+  <details style="margin-top:0.7rem;">
+    <summary style="cursor:pointer;font-size:0.8rem;color:var(--text-muted);">Re-provision (reinstall the steward account + managed key, then the agent)</summary>
+    <p style="font-size:0.78rem;color:var(--text-dim);margin:0.5rem 0;">
+      Use after regenerating the managed key, or if SSH auth as <code>steward</code> breaks.
+      Connects over a one-time bootstrap user + password (not stored).
+    </p>
+    <form method="post" action="/plugins/host_agent/provision"
+          style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;">
+      <input type="hidden" name="inventory_scope" value="{{ scope }}">
+      <div class="form-group" style="margin:0;">
+        <label>Bootstrap user</label>
+        <input type="text" name="bootstrap_user" required placeholder="root" style="width:8rem;" autocomplete="off">
+      </div>
+      <div class="form-group" style="margin:0;">
+        <label>Bootstrap password</label>
+        <input type="password" name="bootstrap_password" required style="width:10rem;" autocomplete="new-password">
+      </div>
+      <button type="submit" class="btn btn-sm">Re-provision</button>
+    </form>
+  </details>
+  {% endif %}
+
+  {% else %}
+  {# ── Not reporting: pending (token minted, no check-in) or never installed ── #}
+  {% if reg %}
+  <div style="background:color-mix(in srgb,var(--yellow) 10%,var(--bg-elevated));
+              border:1px solid color-mix(in srgb,var(--yellow) 30%,var(--border));
+              border-radius:6px;padding:0.7rem 0.9rem;margin-bottom:0.75rem;font-size:0.84rem;">
+    A token was minted for this host but <strong>no metrics have arrived yet</strong> — the
+    deploy may still be running, or it failed. Open the latest
+    <a href="/ansible/">Ansible run</a> to check, then retry below. Live metrics appear here
+    once the agent checks in.
+  </div>
+  {% else %}
+  <p style="color:var(--text-muted);font-size:0.88rem;margin-bottom:0.75rem;">
+    No agent installed. The agent reports CPU, memory, disk, network and more back to Steward.
+  </p>
+  {% endif %}
+
+  {% if session.user_role != 'admin' %}
+  <p style="color:var(--text-dim);font-size:0.85rem;">An admin can install the agent here.</p>
+
+  {% elif not ansible_available %}
+  <p style="color:var(--text-dim);font-size:0.85rem;">
+    Ansible isn't available, so the agent can't be deployed from here. Install it manually with the
+    <a href="/plugins/host_agent/fleet/">curl install command</a>.
+  </p>
+
+  {% elif not managed_key_set %}
+  <div style="background:color-mix(in srgb,var(--yellow) 12%,var(--bg-elevated));
+              border:1px solid color-mix(in srgb,var(--yellow) 35%,var(--border));
+              border-radius:6px;padding:0.7rem 0.9rem;display:flex;align-items:center;gap:0.8rem;flex-wrap:wrap;">
+    <span style="color:var(--yellow);">⚠</span>
+    <span style="font-size:0.84rem;flex:1;min-width:13rem;">No managed SSH key yet — Steward needs one to log into hosts.</span>
+    <form method="post" action="/settings/ansible/generate-key" style="margin:0;">
+      <input type="hidden" name="next" value="/hosts/{{ host.id }}">
+      <button type="submit" class="btn btn-sm">Generate managed key</button>
+    </form>
+  </div>
+
+  {% elif not target %}
+  <p style="color:var(--text-dim);font-size:0.85rem;">
+    Link or create an Ansible target for this host (in the Ansible section below) first — provisioning
+    needs an SSH connection.
+  </p>
+
+  {% else %}
+  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.75rem;">
+    Deploys the agent to <code>{{ target.name }}</code> by running an Ansible playbook. Steward mints
+    the host's API token automatically; you'll watch the run live.
+  </p>
+  {# Provision: brand-new host (creates steward account + key over a one-time password) #}
+  <form method="post" action="/plugins/host_agent/provision"
+        style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;margin-bottom:0.75rem;">
+    <input type="hidden" name="inventory_scope" value="{{ scope }}">
+    <div style="font-size:0.78rem;color:var(--text-muted);width:100%;">Provision (first contact — fresh host):</div>
+    <div class="form-group" style="margin:0;">
+      <label>Bootstrap user</label>
+      <input type="text" name="bootstrap_user" required placeholder="root" style="width:8rem;" autocomplete="off">
+    </div>
+    <div class="form-group" style="margin:0;">
+      <label>Bootstrap password</label>
+      <input type="password" name="bootstrap_password" required style="width:10rem;" autocomplete="new-password">
+    </div>
+    <button type="submit" class="btn btn-sm">Provision</button>
+  </form>
+  {# Install: host already has the steward account (managed key works) #}
+  <form method="post" action="/plugins/host_agent/deploy" style="display:flex;gap:0.6rem;align-items:center;">
+    <input type="hidden" name="inventory_scope" value="{{ scope }}">
+    <span style="font-size:0.78rem;color:var(--text-muted);">Already provisioned?</span>
+    <button type="submit" class="btn btn-sm btn-ghost">Install agent (managed key)</button>
+  </form>
+  {% endif %}
+
+  {% if reg and session.user_role == 'admin' %}
+  <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0.75rem 0 0;"
+        onsubmit="return confirm('Clear this pending registration? Its token is discarded.');">
+    <button type="submit" class="btn btn-sm btn-ghost">Clear pending registration</button>
+  </form>
+  {% endif %}
+  {% endif %}
+</div>

plugins/host_agent/templates/settings_list.html

@@ -1,7 +1,15 @@
 {% extends "base.html" %}
-{% block title %}Host Agent — Steward{% endblock %}
+{% from "_macros.html" import crumbs %}
+{% block title %}Agent fleet — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), ("Agent fleet", "")]) }}{% endblock %}
 {% block content %}
-<h1 class="page-title">Host Agent — Registered Hosts</h1>
+<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1rem;gap:1rem;flex-wrap:wrap;">
+  <h1 class="page-title" style="margin-bottom:0;">Agent fleet</h1>
+</div>
+<p style="color:var(--text-muted);font-size:0.84rem;margin:-0.4rem 0 1.25rem;">
+  Bulk agent operations across your inventory. For a single host, use its
+  <a href="/hosts/">host page</a> — provisioning and metrics live there.
+</p>
 
 {% if new_token %}
 <div class="card" style="background:var(--accent-bg);">
@@ -14,7 +22,7 @@
 {% endif %}
 
 <div class="card">
-  <form method="post" action="/plugins/host_agent/settings/add-host"
+  <form method="post" action="/plugins/host_agent/fleet/add-host"
         style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
     <div class="form-group" style="margin-bottom:0;">
       <label>Host name</label>
@@ -28,6 +36,107 @@
   </form>
 </div>
 
+{% if ansible_available %}
+{% set scope_select %}
+        <label>Target</label>
+        <select name="inventory_scope" required>
+          {% for g in deploy_groups %}<option value="steward:group:{{ g.id }}">Group: {{ g.name }}</option>{% endfor %}
+          {% for t in deploy_targets %}<option value="steward:target:{{ t.id }}">Target: {{ t.name }}</option>{% endfor %}
+        </select>
+{% endset %}
+
+<div class="card">
+  <h3 style="margin-bottom:0.4rem;">Agent lifecycle via Ansible</h3>
+  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.6rem;">
+    These actions run bundled <strong>Ansible playbooks</strong> to deploy and maintain the
+    Steward monitoring agent on your <a href="/ansible/inventory/targets">inventory hosts</a>.
+    Steward generates each host's API token automatically and connects over SSH as the managed
+    <code>steward</code> account — you'll be dropped into the live Ansible run to watch it.
+  </p>
+
+  {% if not managed_key_set %}
+  <div style="background:color-mix(in srgb,var(--yellow) 12%,var(--bg-elevated));
+              border:1px solid color-mix(in srgb,var(--yellow) 35%,var(--border));
+              border-radius:6px;padding:0.7rem 0.9rem;display:flex;align-items:center;
+              gap:0.8rem;flex-wrap:wrap;">
+    <span style="color:var(--yellow);">⚠</span>
+    <span style="font-size:0.84rem;flex:1;min-width:14rem;">
+      No managed SSH key yet — Steward needs one to log into hosts as <code>steward</code>.
+    </span>
+    <form method="post" action="/settings/ansible/generate-key" style="margin:0;">
+      <input type="hidden" name="next" value="/plugins/host_agent/fleet/">
+      <button type="submit" class="btn btn-sm">Generate managed key</button>
+    </form>
+  </div>
+  {% elif not (deploy_targets or deploy_groups) %}
+  <p style="font-size:0.85rem;color:var(--text-dim);margin:0;">
+    No Ansible inventory targets yet. Add some under <a href="/ansible/inventory/targets">Ansible → Inventory</a>.
+  </p>
+  {% endif %}
+</div>
+
+{% if managed_key_set and (deploy_targets or deploy_groups) %}
+<div class="card">
+  <h3 style="margin-bottom:0.4rem;">1 · Provision a fresh host</h3>
+  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.75rem;">
+    First contact for a brand-new host. Connects with a one-time <strong>bootstrap</strong>
+    user + password (used for this run only, never stored), creates the <code>steward</code>
+    login account with passwordless sudo, installs the managed SSH key, then installs the
+    agent. Run this once per host.
+  </p>
+  <form method="post" action="/plugins/host_agent/provision"
+        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
+    <div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Bootstrap user</label>
+      <input type="text" name="bootstrap_user" required placeholder="root" style="width:9rem;" autocomplete="off">
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Bootstrap password</label>
+      <input type="password" name="bootstrap_password" required style="width:11rem;" autocomplete="new-password">
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Report interval (s)</label>
+      <input type="number" name="agent_interval" value="30" min="5" style="width:7rem;">
+    </div>
+    <button type="submit" class="btn">Provision</button>
+  </form>
+</div>
+
+<div class="card">
+  <h3 style="margin-bottom:0.4rem;">2 · Install / enroll agent</h3>
+  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.75rem;">
+    For a host that's already provisioned (has the <code>steward</code> account) but has no
+    agent yet — or to re-enroll one. Connects as <code>steward</code> with the managed key,
+    mints a fresh token, and installs the agent. No <code>curl | sh</code> needed.
+  </p>
+  <form method="post" action="/plugins/host_agent/deploy"
+        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
+    <div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Report interval (s)</label>
+      <input type="number" name="agent_interval" value="30" min="5" style="width:7rem;">
+    </div>
+    <button type="submit" class="btn">Install</button>
+  </form>
+</div>
+
+<div class="card">
+  <h3 style="margin-bottom:0.4rem;">3 · Update agents</h3>
+  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.75rem;">
+    Refresh the agent binary on hosts already running it. Connects as <code>steward</code>
+    with the managed key and restarts the service — the token and config are left untouched,
+    so identity is preserved.
+  </p>
+  <form method="post" action="/plugins/host_agent/update"
+        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
+    <div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
+    <button type="submit" class="btn">Update</button>
+  </form>
+</div>
+{% endif %}
+{% endif %}
+
 <div class="card-flush">
   <table class="table">
     <thead>
@@ -45,12 +154,12 @@
         <td>{{ item.reg.distro or "—" }}</td>
         <td>{{ item.reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") if item.reg.last_seen_at else "never" }}</td>
         <td class="td-actions">
-          <form method="post" action="/plugins/host_agent/settings/{{ item.reg.host_id }}/rotate-token"
+          <form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/rotate-token"
                 style="display:inline;"
                 onsubmit="return confirm('Rotate the token for this host? The existing agent will stop working until reinstalled with the new token.');">
             <button type="submit" class="btn btn-ghost btn-sm">Rotate token</button>
           </form>
-          <form method="post" action="/plugins/host_agent/settings/{{ item.reg.host_id }}/delete"
+          <form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/delete"
                 style="display:inline;"
                 onsubmit="return confirm('Delete this host registration? The agent will be unable to report metrics until re-registered.');">
             <button type="submit" class="btn btn-danger btn-sm">Delete</button>

plugins/host_agent/templates/widget_history.html

@@ -1,24 +1,88 @@
-<div class="widget-history">
-  <h3>{{ host.name }} — last {{ hours }}h</h3>
-  <canvas id="host-agent-chart-{{ host.id }}" width="600" height="200"></canvas>
+{# host_agent history-graph widget — the host-view utilization chart, embedded
+   on a dashboard. Fills the (resizable) panel; epoch-ms linear axis so no
+   Chart.js date adapter is needed. #}
+{% set has_data = host and (series.cpu_pct or series.mem_used_pct or series.disk_root) %}
+{% if not host %}
+<div style="color:var(--text-muted);font-size:0.85rem;padding:0.75rem 0;">
+  No host selected. <strong>Edit</strong> this dashboard and pick a host for this graph.
+</div>
+{% elif not has_data %}
+<div style="color:var(--text-muted);font-size:0.85rem;padding:0.75rem 0;">
+  No agent metrics for <strong>{{ host.name }}</strong> in the last {{ hours }}h yet.
+</div>
+{% else %}
+<div style="display:flex;flex-direction:column;height:100%;min-height:200px;">
+  {# Name the host the graph represents (panel title is generic) + a live range
+     toggle that re-requests this widget without entering edit mode. #}
+  <div style="flex:0 0 auto;display:flex;align-items:center;gap:0.5rem;flex-wrap:wrap;font-size:0.75rem;color:var(--text-muted);margin-bottom:0.3rem;">
+    {% if session.user_id %}
+    <a href="/hosts/{{ host.id }}" style="font-weight:600;">{{ host.name }}</a>
+    {% else %}
+    <strong style="color:var(--text);">{{ host.name }}</strong>
+    {% endif %}
+    <span class="hist-range" style="margin-left:auto;display:inline-flex;border:1px solid var(--border-mid);border-radius:5px;overflow:hidden;">
+      {% for opt in [1, 6, 24] %}
+      <button type="button" onclick="histRange({{ wid }}, '{{ host.id }}', {{ opt }})"
+        style="border:0;cursor:pointer;padding:0.1rem 0.45rem;font-size:0.72rem;
+               background:{% if opt == hours %}var(--accent){% else %}transparent{% endif %};
+               color:{% if opt == hours %}#fff{% else %}var(--text-muted){% endif %};">{{ opt }}h</button>
+      {% endfor %}
+    </span>
+  </div>
+  <div style="flex:1 1 auto;min-height:150px;position:relative;">
+    <canvas id="host-hist-{{ wid }}"></canvas>
+  </div>
+</div>
 <script>
+// Live time-range switch. Rewrites the parent cell's hx-get so the choice sticks
+// across polls (htmx re-reads the attribute each poll), then fetches immediately.
+window.histRange = function (wid, hostId, hours) {
+  var cell = document.getElementById("widget-" + wid);
+  if (!cell) return;
+  var base = (cell.getAttribute("hx-get") || "").split("?")[0];
+  var url = base + "?host_id=" + encodeURIComponent(hostId) + "&hours=" + hours + "&wid=" + wid;
+  cell.setAttribute("hx-get", url);
+  if (window.htmx) window.htmx.ajax("GET", url, { target: cell, swap: "innerHTML" });
+};
 (function () {
-    const ctx = document.getElementById("host-agent-chart-{{ host.id }}").getContext("2d");
-    const series = {{ series|tojson }};
-    new Chart(ctx, {
+  var el = document.getElementById("host-hist-{{ wid }}");
+  if (!el) return;
+  var series = {{ series|tojson }};
+  var fmtTime = function (v) {
+    var d = new Date(v);
+    return ("0" + d.getHours()).slice(-2) + ":" + ("0" + d.getMinutes()).slice(-2);
+  };
+  var ds = [
+    { key: "cpu_pct",      label: "CPU %",    color: "#c8a840" },
+    { key: "mem_used_pct", label: "Mem %",    color: "#4aa86a" },
+    { key: "disk_root",    label: "Disk / %", color: "#c87840" },
+  ];
+  new Chart(el.getContext("2d"), {
     type: "line",
-      data: {
-        datasets: [
-          { label: "CPU %",       data: series.cpu_pct.map(p => ({x: p.t, y: p.v})) },
-          { label: "Mem %",       data: series.mem_used_pct.map(p => ({x: p.t, y: p.v})) },
-          { label: "Disk % worst", data: series.disk_used_pct_worst.map(p => ({x: p.t, y: p.v})) },
-        ],
-      },
+    data: { datasets: ds.map(function (d) {
+      return {
+        label: d.label,
+        data: (series[d.key] || []).map(function (p) { return { x: p[0], y: p[1] }; }),
+        borderColor: d.color, backgroundColor: d.color, borderWidth: 1.5, tension: 0.25,
+      };
+    }) },
     options: {
-        responsive: true,
-        scales: { x: { type: "time" }, y: { min: 0, max: 100 } },
+      responsive: true, maintainAspectRatio: false,
+      interaction: { mode: "index", intersect: false },
+      elements: { point: { radius: 0 } },
+      scales: {
+        x: { type: "linear", ticks: { callback: function (v) { return fmtTime(v); }, maxTicksLimit: 6, color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
+        // Fixed 0–{{ y_max }} ceiling (server snaps the data peak up to the next
+        // 10% band). Steady across refreshes — auto-scaling made the zoom jump on
+        // every poll — while a banded ceiling still keeps the lines filling the panel.
+        y: { min: 0, max: {{ y_max }}, ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
+      },
+      plugins: {
+        legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } },
+        tooltip: { callbacks: { title: function (items) { return items.length ? fmtTime(items[0].parsed.x) : ""; } } },
+      },
     },
   });
 })();
 </script>
-</div>
+{% endif %}

plugins/host_agent/templates/widget_table.html

@@ -1,31 +1,74 @@
-{# host_agent widget — fleet glance, flex rows (no header wrap) #}
+{# host_agent widget — fleet glance. One horizontal row per host, zebra-striped
+   so adjacent hosts read as distinct groupings: health dot + host name on the
+   left, metric cells (cpu/mem/disk/load) each with a trend sparkline laid out
+   horizontally on the right. The name wraps (never hard-truncated) but the row
+   stays horizontal. #}
 {% if rows %}
-<div style="display:grid;gap:0.1rem;">
+{% macro metric_cell(label, value, fmt, svg, style="", title="") %}
+<div style="min-width:62px;flex:0 0 auto;" title="{{ title }}">
+  <div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
+  <div style="font-weight:600;font-size:0.8rem;font-variant-numeric:tabular-nums;{{ style }}">
+    {% if value is not none %}{{ fmt|format(value) }}{% else %}—{% endif %}
+  </div>
+  <div style="line-height:0;height:16px;">{{ svg | safe }}</div>
+</div>
+{% endmacro %}
+{# Human-readable throughput, matching the host-detail page. #}
+{% macro fmt_bps(v) %}
+{%- if v is none -%}—
+{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s
+{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s
+{%- else -%}{{ "%.0f"|format(v) }} B/s
+{%- endif -%}
+{% endmacro %}
+{# Two-line down/up (or read/write) throughput cell. #}
+{% macro io_cell(label, down_sym, down_val, up_sym, up_val, title="") %}
+<div style="min-width:80px;flex:0 0 auto;" title="{{ title }}">
+  <div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
+  <div style="font-size:0.72rem;font-variant-numeric:tabular-nums;line-height:1.3;">
+    <div>{{ down_sym }} {{ fmt_bps(down_val) }}</div>
+    <div>{{ up_sym }} {{ fmt_bps(up_val) }}</div>
+  </div>
+</div>
+{% endmacro %}
+<div class="host-fleet">
   {% for r in rows %}
-  {% set stale = not r.reg.last_seen_at %}
-  <div style="display:flex;align-items:center;gap:0.6rem;font-size:0.82rem;padding:0.3rem 0;border-bottom:1px solid var(--border);">
-    <span class="dot {% if stale %}dot-dim{% elif (r.cpu_pct or 0) >= 90 or (r.mem_used_pct or 0) >= 90 or (r.disk_worst or 0) >= 90 %}dot-warn{% else %}dot-up{% endif %}"></span>
-    <a href="/plugins/host_agent/{{ r.host.id }}/"
-       style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-weight:500;">
-      {{ r.host.name }}
-    </a>
-    <span style="display:flex;gap:0.75rem;font-variant-numeric:tabular-nums;color:var(--text-muted);font-size:0.75rem;flex-shrink:0;">
-      {% if r.cpu_pct is not none %}
-      <span title="CPU"><span style="color:var(--text-dim);">cpu</span> {{ "%.0f"|format(r.cpu_pct) }}%</span>
+  {% set stale = r.stale %}
+  <div style="display:flex;align-items:center;gap:0.6rem 1rem;flex-wrap:wrap;padding:0.5rem 0.6rem;border-radius:4px;{% if loop.index0 % 2 %}background:var(--bg-elevated);{% endif %}">
+    {# Left — health dot + full host name (wraps, never truncated) + last seen #}
+    <div style="display:flex;align-items:center;gap:0.5rem;flex:1 1 150px;min-width:120px;">
+      <span class="dot {% if stale %}dot-dim{% elif (r.cpu_pct or 0) >= 90 or (r.mem_used_pct or 0) >= 90 or (r.disk_worst or 0) >= 90 %}dot-warn{% else %}dot-up{% endif %}"
+            style="flex-shrink:0;"
+            title="{% if stale %}stale / no recent data{% else %}warns at CPU/memory ≥90% or any disk mount ≥90% (the 'disk /' figure shows root only){% endif %}"></span>
+      <span style="min-width:0;">
+        {% if session.user_id %}
+        <a href="/hosts/{{ r.host.id }}" style="font-weight:600;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</a>
+        {% else %}
+        <span style="font-weight:600;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</span>
         {% endif %}
-      {% if r.mem_used_pct is not none %}
-      <span title="Memory"><span style="color:var(--text-dim);">mem</span> {{ "%.0f"|format(r.mem_used_pct) }}%</span>
-      {% endif %}
-      {% if r.disk_worst is not none %}
-      <span title="Worst disk mount"><span style="color:var(--text-dim);">disk</span> {{ "%.0f"|format(r.disk_worst) }}%</span>
-      {% endif %}
-      {% if r.load_1m is not none %}
-      <span title="Load average 1m"><span style="color:var(--text-dim);">load</span> {{ "%.2f"|format(r.load_1m) }}</span>
-      {% endif %}
-    </span>
-    <span style="min-width:60px;text-align:right;font-size:0.72rem;color:var(--text-dim);flex-shrink:0;">
+        <span style="display:block;font-size:0.7rem;color:var(--text-dim);">
           {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }}
         </span>
+      </span>
+    </div>
+    {# Right — metric cells with value + trend, laid out horizontally #}
+    <div style="display:flex;flex-wrap:wrap;gap:0.4rem 1rem;justify-content:flex-end;">
+      {{ metric_cell("cpu", r.cpu_pct, "%.0f%%", r.sparks.cpu, threshold_style(r.cpu_pct, 'cpu'), "Average CPU utilization") }}
+      {{ metric_cell("mem", r.mem_used_pct, "%.0f%%", r.sparks.mem, threshold_style(r.mem_used_pct, 'mem'), "Memory in use") }}
+      {{ metric_cell("disk /", r.disk_root, "%.0f%%", r.sparks.disk, threshold_style(r.disk_root, 'disk'), "Root filesystem (/) usage") }}
+      {{ metric_cell("load", r.load_1m, "%.2f", r.sparks.load, "", "Load average (1m)") }}
+      {# Extra agent metrics — shown only when the host reports them (VMs/containers
+         often have no temp sensor, etc.) so absent data doesn't clutter the row. #}
+      {% if r.net_rx_bps is not none or r.net_tx_bps is not none %}
+      {{ io_cell("net", "↓", r.net_rx_bps, "↑", r.net_tx_bps, "Network throughput (down / up)") }}
+      {% endif %}
+      {% if r.disk_read_bps is not none or r.disk_write_bps is not none %}
+      {{ io_cell("disk i/o", "rd", r.disk_read_bps, "wr", r.disk_write_bps, "Disk I/O (read / write)") }}
+      {% endif %}
+      {% if r.temp_max is not none %}
+      {{ metric_cell("temp", r.temp_max, "%.0f°C", "", threshold_style(r.temp_max, 'temp'), "Hottest sensor") }}
+      {% endif %}
+    </div>
   </div>
   {% endfor %}
 </div>

plugins/http/__init__.py

@@ -1,24 +0,0 @@
-# plugins/http/__init__.py
-from __future__ import annotations
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from quart import Quart
-
-_app: "Quart | None" = None
-
-
-def setup(app: "Quart") -> None:
-    global _app
-    _app = app
-    from .models import HttpMonitor, HttpResult  # noqa: F401
-
-
-def get_scheduled_tasks() -> list:
-    from .scheduler import make_task
-    return [make_task(_app)]
-
-
-def get_blueprint():
-    from .routes import http_bp
-    return http_bp

plugins/http/migrations/env.py

@@ -1,70 +0,0 @@
-# plugins/http/migrations/env.py
-"""Alembic env.py for the HTTP monitoring plugin."""
-import asyncio
-import os
-import sys
-from pathlib import Path
-from logging.config import fileConfig
-
-from sqlalchemy import pool
-from sqlalchemy.engine import Connection
-from sqlalchemy.ext.asyncio import async_engine_from_config
-from alembic import context
-
-sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
-
-from steward.models.base import Base
-import steward.models  # noqa: F401
-from plugins.http.models import HttpMonitor, HttpResult  # noqa: F401
-
-config = context.config
-if config.config_file_name is not None:
-    fileConfig(config.config_file_name)
-
-target_metadata = Base.metadata
-
-
-def _get_url() -> str:
-    import yaml
-    cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml")
-    try:
-        with open(cfg_path) as f:
-            cfg = yaml.safe_load(f) or {}
-        url = cfg.get("database", {}).get("url", "")
-    except FileNotFoundError:
-        url = ""
-    return os.environ.get("STEWARD_DATABASE__URL", url)
-
-
-def run_migrations_offline() -> None:
-    context.configure(
-        url=_get_url(), target_metadata=target_metadata,
-        literal_binds=True, dialect_opts={"paramstyle": "named"},
-    )
-    with context.begin_transaction():
-        context.run_migrations()
-
-
-def do_run_migrations(connection: Connection) -> None:
-    context.configure(connection=connection, target_metadata=target_metadata)
-    with context.begin_transaction():
-        context.run_migrations()
-
-
-async def run_async_migrations() -> None:
-    cfg = config.get_section(config.config_ini_section, {})
-    cfg["sqlalchemy.url"] = _get_url()
-    connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool)
-    async with connectable.connect() as connection:
-        await connection.run_sync(do_run_migrations)
-    await connectable.dispose()
-
-
-def run_migrations_online() -> None:
-    asyncio.run(run_async_migrations())
-
-
-if context.is_offline_mode():
-    run_migrations_offline()
-else:
-    run_migrations_online()

plugins/http/models.py

@@ -1,63 +0,0 @@
-# plugins/http/models.py
-from __future__ import annotations
-import uuid
-from datetime import datetime, timezone
-from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text
-from sqlalchemy.orm import Mapped, mapped_column
-from steward.models.base import Base
-
-
-class HttpMonitor(Base):
-    """A configured HTTP endpoint to check periodically."""
-    __tablename__ = "http_monitors"
-
-    id: Mapped[str] = mapped_column(
-        String(36), primary_key=True, default=lambda: str(uuid.uuid4())
-    )
-    name: Mapped[str] = mapped_column(String(128), nullable=False)
-    url: Mapped[str] = mapped_column(String(2048), nullable=False)
-    method: Mapped[str] = mapped_column(String(8), nullable=False, default="GET")
-    expected_status: Mapped[int] = mapped_column(Integer, nullable=False, default=200)
-    # Optional substring to find in the response body (empty = skip check)
-    content_match: Mapped[str] = mapped_column(String(512), nullable=False, default="")
-    # JSON dict of extra request headers, e.g. {"Authorization": "Bearer ..."}
-    headers_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
-    timeout_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=10)
-    # 0 = use global plugin check_interval_seconds
-    check_interval_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-    follow_redirects: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
-    verify_ssl: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
-    enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
-    # Updated after each check so the scheduler can compute next-run time efficiently
-    last_checked_at: Mapped[datetime | None] = mapped_column(
-        DateTime(timezone=True), nullable=True
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False,
-        default=lambda: datetime.now(timezone.utc),
-    )
-
-
-class HttpResult(Base):
-    """One check result for an HttpMonitor."""
-    __tablename__ = "http_results"
-
-    id: Mapped[str] = mapped_column(
-        String(36), primary_key=True, default=lambda: str(uuid.uuid4())
-    )
-    monitor_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
-    checked_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False,
-        default=lambda: datetime.now(timezone.utc),
-        index=True,
-    )
-    status_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
-    response_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
-    is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-    # None when no content_match is configured; True/False when it is
-    content_matched: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
-    error_msg: Mapped[str | None] = mapped_column(String(512), nullable=True)
-    # TLS expiry if the endpoint is HTTPS; None for HTTP or on error
-    tls_expires_at: Mapped[datetime | None] = mapped_column(
-        DateTime(timezone=True), nullable=True
-    )

plugins/http/plugin.yaml

@@ -1,19 +0,0 @@
-name: http
-version: "1.0.0"
-description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry"
-author: "Steward"
-license: "MIT"
-min_app_version: "0.1.0"
-repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
-homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/http"
-tags:
-  - monitoring
-  - http
-  - synthetic
-  - uptime
-
-config:
-  check_interval_seconds: 60
-  default_timeout_seconds: 10
-  follow_redirects: true
-  verify_ssl: true

plugins/http/routes.py

@@ -1,228 +0,0 @@
-# plugins/http/routes.py
-from __future__ import annotations
-import uuid
-from datetime import datetime, timezone
-from quart import Blueprint, current_app, render_template, request, redirect, url_for
-from sqlalchemy import Integer, cast, func, select
-
-from steward.auth.middleware import require_role
-from steward.models.users import UserRole
-from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds
-from .models import HttpMonitor, HttpResult
-
-http_bp = Blueprint("http", __name__, template_folder="templates")
-
-
-def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
-    if len(values) < 2:
-        return f'<svg width="{width}" height="{height}"></svg>'
-    mn, mx = min(values), max(values)
-    if mx == mn:
-        mx = mn + 1.0
-    step = width / (len(values) - 1)
-    pts = []
-    for i, v in enumerate(values):
-        x = i * step
-        y = height - (v - mn) / (mx - mn) * (height - 2) - 1
-        pts.append(f"{x:.1f},{y:.1f}")
-    poly = " ".join(pts)
-    return (
-        f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
-        f'style="vertical-align:middle;">'
-        f'<polyline points="{poly}" fill="none" stroke="#6060c0" stroke-width="1.5"/>'
-        f'</svg>'
-    )
-
-
-async def _render_rows(range_key: str = DEFAULT_RANGE):
-    since, range_key = parse_range(range_key)
-    b_secs = bucket_seconds(since)
-    bucket_col = (
-        cast(func.extract('epoch', HttpResult.checked_at), Integer) / b_secs
-    ).label("bucket")
-
-    async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(HttpMonitor).order_by(HttpMonitor.created_at)
-        )
-        monitors = list(result.scalars())
-
-        # Latest result per monitor
-        latest_map: dict[str, HttpResult] = {}
-        histories: dict[str, list] = {}
-        for m in monitors:
-            r = await db.execute(
-                select(HttpResult)
-                .where(HttpResult.monitor_id == m.id)
-                .order_by(HttpResult.checked_at.desc())
-                .limit(1)
-            )
-            latest = r.scalar_one_or_none()
-            if latest:
-                latest_map[m.id] = latest
-
-            r2 = await db.execute(
-                select(
-                    func.avg(HttpResult.response_ms).label("response_ms"),
-                    func.min(HttpResult.is_up.cast(Integer)).label("had_down"),
-                    bucket_col,
-                )
-                .where(HttpResult.monitor_id == m.id)
-                .where(HttpResult.checked_at >= since)
-                .group_by(bucket_col)
-                .order_by(bucket_col)
-            )
-            histories[m.id] = r2.all()
-
-    monitor_data = []
-    for m in monitors:
-        hist = histories.get(m.id, [])
-        latest = latest_map.get(m.id)
-        now = datetime.now(timezone.utc)
-        tls_days = None
-        if latest and latest.tls_expires_at:
-            tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0
-        monitor_data.append({
-            "monitor": m,
-            "latest": latest,
-            "tls_days": tls_days,
-            "sparkline_ms": _sparkline([r.response_ms or 0 for r in hist]),
-        })
-
-    up = sum(1 for d in monitor_data if d["latest"] and d["latest"].is_up)
-    down = sum(1 for d in monitor_data if d["latest"] and not d["latest"].is_up)
-    return monitor_data, up, down, range_key
-
-
-@http_bp.get("/")
-@require_role(UserRole.viewer)
-async def index():
-    poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
-    current_range = request.args.get("range", DEFAULT_RANGE)
-    return await render_template(
-        "http/index.html",
-        poll_interval=poll_interval,
-        current_range=current_range,
-    )
-
-
-@http_bp.get("/rows")
-@require_role(UserRole.viewer)
-async def rows():
-    """HTMX fragment: monitor list with status + sparklines."""
-    monitor_data, up, down, range_key = await _render_rows(
-        request.args.get("range", DEFAULT_RANGE)
-    )
-    return await render_template(
-        "http/rows.html",
-        monitor_data=monitor_data,
-        up=up,
-        down=down,
-        range_key=range_key,
-    )
-
-
-@http_bp.post("/add")
-@require_role(UserRole.operator)
-async def add_monitor():
-    """Add a new HTTP monitor."""
-    form = await request.form
-    url = form.get("url", "").strip()
-    if not url:
-        return redirect(url_for("http.index"))
-    if not url.startswith(("http://", "https://")):
-        url = "https://" + url
-
-    monitor = HttpMonitor(
-        id=str(uuid.uuid4()),
-        name=form.get("name", "").strip() or url,
-        url=url,
-        method=form.get("method", "GET").upper(),
-        expected_status=int(form.get("expected_status", 200) or 200),
-        content_match=form.get("content_match", "").strip(),
-        headers_json="{}",
-        timeout_seconds=int(form.get("timeout_seconds", 10) or 10),
-        check_interval_seconds=int(form.get("check_interval_seconds", 0) or 0),
-        follow_redirects="follow_redirects" in form,
-        verify_ssl="verify_ssl" in form,
-        enabled=True,
-        created_at=datetime.now(timezone.utc),
-    )
-    async with current_app.db_sessionmaker() as db:
-        async with db.begin():
-            db.add(monitor)
-
-    return redirect(url_for("http.index"))
-
-
-@http_bp.post("/<monitor_id>/delete")
-@require_role(UserRole.operator)
-async def delete_monitor(monitor_id: str):
-    async with current_app.db_sessionmaker() as db:
-        async with db.begin():
-            m = await db.get(HttpMonitor, monitor_id)
-            if m:
-                await db.delete(m)
-    return redirect(url_for("http.index"))
-
-
-@http_bp.post("/<monitor_id>/toggle")
-@require_role(UserRole.operator)
-async def toggle_monitor(monitor_id: str):
-    async with current_app.db_sessionmaker() as db:
-        async with db.begin():
-            m = await db.get(HttpMonitor, monitor_id)
-            if m:
-                m.enabled = not m.enabled
-    return redirect(url_for("http.index"))
-
-
-@http_bp.get("/widget")
-@require_role(UserRole.viewer)
-async def widget():
-    """HTMX dashboard widget: up/down counts + monitor list."""
-    show_down_only = request.args.get("show_down_only", "no") == "yes"
-    limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
-    widget_id = request.args.get("wid", "0")
-
-    async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(HttpMonitor)
-            .where(HttpMonitor.enabled == True)  # noqa: E712
-            .order_by(HttpMonitor.created_at)
-        )
-        monitors = list(result.scalars())
-
-        latest_map: dict[str, HttpResult] = {}
-        for m in monitors:
-            r = await db.execute(
-                select(HttpResult)
-                .where(HttpResult.monitor_id == m.id)
-                .order_by(HttpResult.checked_at.desc())
-                .limit(1)
-            )
-            latest = r.scalar_one_or_none()
-            if latest:
-                latest_map[m.id] = latest
-
-    monitor_data = [
-        {"monitor": m, "latest": latest_map.get(m.id)}
-        for m in monitors
-    ]
-
-    up = sum(1 for d in monitor_data if d["latest"] and d["latest"].is_up)
-    down = sum(1 for d in monitor_data if d["latest"] and not d["latest"].is_up)
-    pending = sum(1 for d in monitor_data if not d["latest"])
-
-    if show_down_only:
-        monitor_data = [d for d in monitor_data if not (d["latest"] and d["latest"].is_up)]
-
-    return await render_template(
-        "http/widget.html",
-        monitor_data=monitor_data[:limit],
-        up=up,
-        down=down,
-        pending=pending,
-        show_down_only=show_down_only,
-        widget_id=widget_id,
-    )

plugins/http/scheduler.py

@@ -1,121 +0,0 @@
-# plugins/http/scheduler.py
-from __future__ import annotations
-import asyncio
-import json
-import logging
-from datetime import datetime, timezone
-
-from steward.core.scheduler import ScheduledTask
-from steward.core.alerts import record_metric
-
-logger = logging.getLogger(__name__)
-
-
-def make_task(app) -> ScheduledTask:
-    interval = int(
-        app.config["PLUGINS"]["http"].get("check_interval_seconds", 60)
-    )
-
-    async def check():
-        await _do_checks(app)
-
-    return ScheduledTask(
-        name="http_check",
-        coro_factory=check,
-        interval_seconds=interval,
-        run_on_startup=True,
-    )
-
-
-async def _do_checks(app) -> None:
-    from sqlalchemy import select
-    from .models import HttpMonitor, HttpResult
-    from .checker import run_check
-
-    cfg = app.config["PLUGINS"]["http"]
-    global_interval = int(cfg.get("check_interval_seconds", 60))
-    global_timeout = int(cfg.get("default_timeout_seconds", 10))
-    global_follow = bool(cfg.get("follow_redirects", True))
-    global_verify = bool(cfg.get("verify_ssl", True))
-
-    now = datetime.now(timezone.utc)
-
-    async with app.db_sessionmaker() as session:
-        result = await session.execute(
-            select(HttpMonitor).where(HttpMonitor.enabled == True)  # noqa: E712
-        )
-        monitors = list(result.scalars())
-
-    # Filter to monitors whose next check time has passed
-    due: list[HttpMonitor] = []
-    for m in monitors:
-        effective_interval = m.check_interval_seconds or global_interval
-        if m.last_checked_at is None:
-            due.append(m)
-        elif (now - m.last_checked_at).total_seconds() >= effective_interval:
-            due.append(m)
-
-    if not due:
-        return
-
-    async def _check_one(monitor: HttpMonitor) -> tuple[HttpMonitor, dict]:
-        try:
-            headers = json.loads(monitor.headers_json or "{}")
-        except (ValueError, TypeError):
-            headers = {}
-        result = await run_check(
-            url=monitor.url,
-            method=monitor.method,
-            expected_status=monitor.expected_status,
-            content_match=monitor.content_match,
-            headers=headers,
-            timeout_seconds=monitor.timeout_seconds or global_timeout,
-            follow_redirects=monitor.follow_redirects if monitor.follow_redirects is not None else global_follow,
-            verify_ssl=monitor.verify_ssl if monitor.verify_ssl is not None else global_verify,
-        )
-        return monitor, result
-
-    pairs = await asyncio.gather(*[_check_one(m) for m in due], return_exceptions=True)
-
-    check_time = datetime.now(timezone.utc)
-
-    async with app.db_sessionmaker() as session:
-        async with session.begin():
-            for item in pairs:
-                if isinstance(item, Exception):
-                    logger.error("HTTP check task error: %s", item)
-                    continue
-                monitor, res = item
-
-                session.add(HttpResult(
-                    monitor_id=monitor.id,
-                    checked_at=check_time,
-                    status_code=res["status_code"],
-                    response_ms=res["response_ms"],
-                    is_up=res["is_up"],
-                    content_matched=res["content_matched"],
-                    error_msg=res["error_msg"],
-                    tls_expires_at=res["tls_expires_at"],
-                ))
-
-                # Update monitor's last_checked_at for next-run scheduling
-                db_monitor = await session.get(HttpMonitor, monitor.id)
-                if db_monitor:
-                    db_monitor.last_checked_at = check_time
-
-                # Alert pipeline
-                if res["response_ms"] is not None:
-                    await record_metric(
-                        session=session,
-                        source_module="http",
-                        resource_name=monitor.name,
-                        metric_name="response_ms",
-                        value=res["response_ms"],
-                    )
-                await record_metric(
-                    session=session,
-                    source_module="http",
-                    resource_name=monitor.name,
-                    metric_name="is_up",
-                    value=1.0 if res["is_up"] else 0.0,
-                )

plugins/http/templates/http/index.html

@@ -1,77 +0,0 @@
-{% extends "base.html" %}
-{% block title %}HTTP Monitors — Steward{% endblock %}
-{% block content %}
-<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1.5rem;gap:1rem;flex-wrap:wrap;">
-  <h1 class="page-title" style="margin-bottom:0;">HTTP Monitors</h1>
-  {% include "_time_range.html" %}
-</div>
-
-{# ── Add monitor form ─────────────────────────────────────────────────────── #}
-<div class="card" style="margin-bottom:1.25rem;">
-  <div class="section-title" style="margin-bottom:0.75rem;">Add Monitor</div>
-  <form method="post" action="/plugins/http/add" style="display:grid;gap:0.6rem;">
-    <div style="display:grid;grid-template-columns:1fr 1fr;gap:0.6rem;">
-      <div>
-        <label style="font-size:0.75rem;color:var(--text-muted);display:block;margin-bottom:0.2rem;">Name</label>
-        <input type="text" name="name" placeholder="My Service" autocomplete="off"
-               style="width:100%;padding:0.4rem 0.65rem;background:var(--bg);border:1px solid var(--border-mid);border-radius:4px;color:var(--text);font-size:0.88rem;">
-      </div>
-      <div>
-        <label style="font-size:0.75rem;color:var(--text-muted);display:block;margin-bottom:0.2rem;">URL <span style="color:var(--red);">*</span></label>
-        <input type="text" name="url" placeholder="https://example.com" autocomplete="off" required
-               style="width:100%;padding:0.4rem 0.65rem;background:var(--bg);border:1px solid var(--border-mid);border-radius:4px;color:var(--text);font-size:0.88rem;">
-      </div>
-    </div>
-    <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(140px,1fr));gap:0.6rem;align-items:end;">
-      <div>
-        <label style="font-size:0.75rem;color:var(--text-muted);display:block;margin-bottom:0.2rem;">Method</label>
-        <select name="method" style="width:100%;padding:0.4rem 0.65rem;background:var(--bg);border:1px solid var(--border-mid);border-radius:4px;color:var(--text);font-size:0.88rem;">
-          <option value="GET">GET</option>
-          <option value="HEAD">HEAD</option>
-          <option value="POST">POST</option>
-        </select>
-      </div>
-      <div>
-        <label style="font-size:0.75rem;color:var(--text-muted);display:block;margin-bottom:0.2rem;">Expected status</label>
-        <input type="number" name="expected_status" value="200" min="100" max="599"
-               style="width:100%;padding:0.4rem 0.65rem;background:var(--bg);border:1px solid var(--border-mid);border-radius:4px;color:var(--text);font-size:0.88rem;">
-      </div>
-      <div>
-        <label style="font-size:0.75rem;color:var(--text-muted);display:block;margin-bottom:0.2rem;">Timeout (s)</label>
-        <input type="number" name="timeout_seconds" value="10" min="1" max="60"
-               style="width:100%;padding:0.4rem 0.65rem;background:var(--bg);border:1px solid var(--border-mid);border-radius:4px;color:var(--text);font-size:0.88rem;">
-      </div>
-      <div>
-        <label style="font-size:0.75rem;color:var(--text-muted);display:block;margin-bottom:0.2rem;">Interval (s, 0=global)</label>
-        <input type="number" name="check_interval_seconds" value="0" min="0"
-               style="width:100%;padding:0.4rem 0.65rem;background:var(--bg);border:1px solid var(--border-mid);border-radius:4px;color:var(--text);font-size:0.88rem;">
-      </div>
-      <div>
-        <label style="font-size:0.75rem;color:var(--text-muted);display:block;margin-bottom:0.2rem;">Content match (substring)</label>
-        <input type="text" name="content_match" placeholder="optional"
-               style="width:100%;padding:0.4rem 0.65rem;background:var(--bg);border:1px solid var(--border-mid);border-radius:4px;color:var(--text);font-size:0.88rem;">
-      </div>
-      <div style="display:flex;flex-direction:column;gap:0.3rem;padding-top:0.8rem;">
-        <label style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;cursor:pointer;">
-          <input type="checkbox" name="follow_redirects" checked> Follow redirects
-        </label>
-        <label style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;cursor:pointer;">
-          <input type="checkbox" name="verify_ssl" checked> Verify SSL
-        </label>
-      </div>
-    </div>
-    <div>
-      <button type="submit" class="btn btn-sm">Add Monitor</button>
-    </div>
-  </form>
-</div>
-
-{# ── Monitor list ─────────────────────────────────────────────────────────── #}
-<div id="http-rows"
-     hx-get="/plugins/http/rows"
-     hx-trigger="load, every {{ poll_interval }}s, rangeChange from:body"
-     hx-include="#time-range"
-     hx-swap="innerHTML">
-  <div style="color:var(--text-muted);font-size:0.9rem;padding:2rem;">Loading...</div>
-</div>
-{% endblock %}

plugins/http/templates/http/rows.html

@@ -1,109 +0,0 @@
-{# http/rows.html — HTMX fragment for HTTP monitor status list #}
-
-{# ── Summary ─────────────────────────────────────────────────────────────── #}
-<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(130px,1fr));gap:0.75rem;margin-bottom:1.25rem;">
-  <div class="card" style="margin-bottom:0;">
-    <div class="section-title" style="margin-bottom:0.3rem;">Up</div>
-    <span class="stat-val" style="color:var(--green);">{{ up }}</span>
-  </div>
-  <div class="card" style="margin-bottom:0;">
-    <div class="section-title" style="margin-bottom:0.3rem;">Down</div>
-    <span class="stat-val" style="color:{% if down %}var(--red){% else %}var(--text-muted){% endif %};">{{ down }}</span>
-  </div>
-  <div class="card" style="margin-bottom:0;">
-    <div class="section-title" style="margin-bottom:0.3rem;">Monitors</div>
-    <span class="stat-val">{{ monitor_data | length }}</span>
-  </div>
-</div>
-
-{# ── Monitor list ─────────────────────────────────────────────────────────── #}
-{% if monitor_data %}
-<div class="card-flush">
-  <table class="table">
-    <thead>
-      <tr>
-        <th>Monitor</th>
-        <th>Status</th>
-        <th>Response</th>
-        <th style="min-width:90px;">History</th>
-        <th>TLS expiry</th>
-        <th style="width:1%;"></th>
-      </tr>
-    </thead>
-    <tbody>
-      {% for item in monitor_data %}
-      {% set m = item.monitor %}
-      {% set latest = item.latest %}
-      <tr>
-        <td>
-          <div style="font-weight:500;font-size:0.9rem;">{{ m.name }}</div>
-          <div style="font-size:0.75rem;color:var(--text-muted);font-family:ui-monospace,monospace;word-break:break-all;">
-            <span style="color:var(--text-dim);">{{ m.method }}</span> {{ m.url }}
-          </div>
-          {% if not m.enabled %}
-          <span style="font-size:0.72rem;color:var(--text-dim);">paused</span>
-          {% endif %}
-        </td>
-        <td>
-          {% if not latest %}
-          <span style="color:var(--text-dim);font-size:0.82rem;">pending</span>
-          {% elif latest.is_up %}
-          <div style="display:flex;align-items:center;gap:0.4rem;">
-            <span class="dot dot-up"></span>
-            <span style="font-size:0.82rem;color:var(--green);">{{ latest.status_code }}</span>
-          </div>
-          {% else %}
-          <div style="display:flex;align-items:center;gap:0.4rem;">
-            <span class="dot dot-down"></span>
-            <span style="font-size:0.82rem;color:var(--red);">
-              {{ latest.status_code or latest.error_msg or "error" }}
-            </span>
-          </div>
-          {% if latest.content_matched == false %}
-          <div style="font-size:0.72rem;color:var(--orange);">content mismatch</div>
-          {% endif %}
-          {% endif %}
-        </td>
-        <td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
-          {% if latest and latest.response_ms is not none %}
-          <span style="color:{% if latest.response_ms > 2000 %}var(--red){% elif latest.response_ms > 500 %}var(--orange){% else %}var(--text){% endif %};">
-            {{ "%.0f" | format(latest.response_ms) }}ms
-          </span>
-          {% else %}
-          <span style="color:var(--text-dim);">—</span>
-          {% endif %}
-        </td>
-        <td>{{ item.sparkline_ms | safe }}</td>
-        <td style="font-size:0.82rem;">
-          {% if item.tls_days is not none %}
-          <span style="color:{% if item.tls_days < 14 %}var(--red){% elif item.tls_days < 30 %}var(--orange){% else %}var(--text-muted){% endif %};">
-            {{ item.tls_days | int }}d
-          </span>
-          {% elif m.url.startswith('https://') %}
-          <span style="color:var(--text-dim);">—</span>
-          {% endif %}
-        </td>
-        <td style="white-space:nowrap;">
-          <form method="post" action="/plugins/http/{{ m.id }}/toggle" style="display:inline;margin:0;">
-            <button type="submit" class="btn btn-ghost btn-sm" style="font-size:0.72rem;">
-              {% if m.enabled %}Pause{% else %}Resume{% endif %}
-            </button>
-          </form>
-          <form method="post" action="/plugins/http/{{ m.id }}/delete"
-                style="display:inline;margin:0;"
-                onsubmit="return confirm('Delete {{ m.name }}?')">
-            <button type="submit" class="btn btn-danger btn-sm" style="font-size:0.72rem;">Del</button>
-          </form>
-        </td>
-      </tr>
-      {% endfor %}
-    </tbody>
-  </table>
-</div>
-{% else %}
-<div class="card" style="text-align:center;padding:3rem;">
-  <div style="color:var(--text-muted);font-size:0.9rem;">
-    No monitors configured yet. Add one using the form above.
-  </div>
-</div>
-{% endif %}

plugins/http/templates/http/widget.html

@@ -1,56 +0,0 @@
-{# http/widget.html — dashboard widget: HTTP monitor summary #}
-{% if not monitor_data and up == 0 and down == 0 and pending == 0 %}
-<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">
-  No monitors configured. <a href="/plugins/http/">Add monitors →</a>
-</div>
-{% else %}
-
-<div style="display:flex;gap:1rem;margin-bottom:0.65rem;flex-wrap:wrap;">
-  {% if up %}
-  <div>
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--green);">{{ up }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">up</span>
-  </div>
-  {% endif %}
-  {% if down %}
-  <div>
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--red);">{{ down }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">down</span>
-  </div>
-  {% endif %}
-  {% if pending %}
-  <div>
-    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--text-dim);">{{ pending }}</span>
-    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">pending</span>
-  </div>
-  {% endif %}
-</div>
-
-<div style="display:grid;gap:3px;">
-  {% for item in monitor_data %}
-  {% set latest = item.latest %}
-  <div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
-    {% if not latest %}
-    <span class="dot dot-dim"></span>
-    {% elif latest.is_up %}
-    <span class="dot dot-up"></span>
-    {% else %}
-    <span class="dot dot-down"></span>
-    {% endif %}
-    <span style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
-      {{ item.monitor.name }}
-    </span>
-    {% if latest and latest.response_ms is not none %}
-    <span style="font-size:0.72rem;font-family:ui-monospace,monospace;color:var(--text-muted);flex-shrink:0;">
-      {{ "%.0f" | format(latest.response_ms) }}ms
-    </span>
-    {% elif latest and not latest.is_up %}
-    <span style="font-size:0.72rem;color:var(--red);flex-shrink:0;">
-      {{ latest.status_code or "err" }}
-    </span>
-    {% endif %}
-  </div>
-  {% endfor %}
-</div>
-
-{% endif %}

plugins/index.yaml

@@ -13,31 +13,15 @@ updated: "2026-03-22"
 
 plugins:
 
-  - name: http
-    version: "1.0.0"
-    description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry"
-    author: "Steward"
-    license: "MIT"
-    min_app_version: "0.1.0"
-    repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
-    homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/http"
-    download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/http-v1.0.0/http.zip"
-    checksum_sha256: ""
-    tags:
-      - monitoring
-      - http
-      - synthetic
-      - uptime
-
   - name: docker
-    version: "1.0.0"
-    description: "Docker container status, resource usage, and restart tracking via Docker socket"
+    version: "2.0.0"
+    description: "Per-host Docker container status + resource usage, collected by the Steward host agent"
     author: "Steward"
     license: "MIT"
     min_app_version: "0.1.0"
     repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
     homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker"
-    download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v1.0.0/docker.zip"
+    download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v2.0.0/docker.zip"
     checksum_sha256: ""
     tags:
       - containers

plugins/snmp/__init__.py

@@ -21,3 +21,8 @@ def get_scheduled_tasks() -> list:
 def get_blueprint():
     from .routes import snmp_bp
     return snmp_bp
+
+
+def get_nav() -> list[dict]:
+    # Surfaces in the sidebar's Infrastructure group (SNMP device readings).
+    return [{"label": "SNMP", "href": "/plugins/snmp/"}]

plugins/snmp/plugin.yaml

@@ -15,6 +15,16 @@ tags:
 config:
   poll_interval_seconds: 60
   # Each device is polled independently. OIDs must resolve to numeric values.
+  #
+  # `host` is the SNMP poll target (IP/hostname we query). A device also shows
+  # up on a Steward Host's detail page when it maps to that host. By default
+  # that mapping is implicit: `host` is matched against the Host's address or
+  # name. To bind explicitly instead — e.g. when you poll by IP but the Host is
+  # recorded by DNS name, or for an SNMP-only switch/PDU/UPS — add ONE of:
+  #   host_id:      "<steward-host-uuid>"   # exact Host.id match
+  #   steward_host: "switch01"              # Host name OR address (case-insensitive)
+  # An explicit binding is exclusive: the device then maps ONLY to that host,
+  # never implicitly to another by a coincidental address string.
   devices:
     - name: "core-switch"
       host: "192.168.1.1"

plugins/snmp/poller.py

@@ -1,11 +1,13 @@
 # plugins/snmp/poller.py
 """
-Synchronous SNMP GET helper, run via executor.
+Asynchronous SNMP GET helper.
 
-Requires pysnmp-lextudio (maintained pysnmp fork):
+Requires pysnmp-lextudio (the maintained pysnmp fork), bundled into the Docker
+image via the `snmp` extra (`pip install .[snmp]`):
     pip install 'steward[snmp]'
 
-If pysnmp is not installed, poll_device() returns an empty dict and logs a warning.
+If pysnmp is not installed, poll_device() returns an empty dict and logs a
+warning — SNMP polling is then simply disabled, nothing else breaks.
 """
 from __future__ import annotations
 import logging
@@ -22,39 +24,114 @@ def _pysnmp_available() -> bool:
 
 
 def _mp_model(version: str) -> int:
-    """Map version string to pysnmp mpModel integer."""
+    """Map an SNMP version string to pysnmp's mpModel int (0 = v1, 1 = v2c)."""
     return 0 if version == "1" else 1
 
 
-def poll_device_sync(
+def _close_engine(engine) -> None:
+    """Release the engine's UDP transport socket.
+
+    pysnmp opens a UDP socket per ``SnmpEngine`` and never closes it on its own.
+    Since we build a fresh engine for every poll, an unclosed engine leaks one
+    file descriptor each scheduler tick; over hours of polling that exhausts the
+    process fd limit (``OSError: [Errno 24] Too many open files``), which also
+    takes down the app's listening socket. So close it explicitly here.
+
+    The close method/attribute names differ across pysnmp majors (6.2.x lextudio
+    is camelCase, canonical 7.x is snake_case), so probe for whatever exists.
+    All paths are best-effort — a failed close must never break the poll loop.
+    """
+    # 7.x exposes a convenience close directly on the engine.
+    for meth in ("close_dispatcher", "closeDispatcher"):
+        fn = getattr(engine, meth, None)
+        if callable(fn):
+            try:
+                fn()
+            except Exception:
+                pass
+            return
+    # 6.2.x: go through the transport dispatcher.
+    for attr in ("transport_dispatcher", "transportDispatcher"):
+        dispatcher = getattr(engine, attr, None)
+        if dispatcher is None:
+            continue
+        for meth in ("close_dispatcher", "closeDispatcher"):
+            fn = getattr(dispatcher, meth, None)
+            if callable(fn):
+                try:
+                    fn()
+                except Exception:
+                    pass
+                return
+        return
+
+
+async def poll_device(
     host: str,
     port: int,
     community: str,
     version: str,
     oids: list[dict],
 ) -> dict[str, float]:
-    """
-    Perform SNMP GET for each OID and return {label: float_value}.
-    Non-numeric OIDs (strings, etc.) are skipped.
-    Returns empty dict on any error.
+    """Perform an SNMP GET for each OID and return ``{label: float_value}``.
+
+    Non-numeric OIDs (strings, etc.) are skipped. Returns an empty dict on any
+    error (unreachable host, wrong community, …) so a flaky device never breaks
+    the poll loop.
+
+    pysnmp's HLAPI is asyncio-only as of v6. The import path moved between major
+    versions, so we support both rather than let a dependency bump silently
+    re-break polling:
+      • pysnmp-lextudio 6.2.x → ``pysnmp.hlapi.asyncio`` (``getCmd`` + a directly
+        constructed ``UdpTransportTarget``).
+      • canonical pysnmp 7.x  → ``pysnmp.hlapi.v3arch.asyncio`` (``get_cmd`` + the
+        async ``UdpTransportTarget.create``).
     """
     if not _pysnmp_available():
         logger.warning("pysnmp not installed — SNMP polling disabled. "
                        "Install with: pip install 'steward[snmp]'")
         return {}
 
-    from pysnmp.hlapi import (
+    try:
+        # canonical pysnmp 7.x
+        from pysnmp.hlapi.v3arch.asyncio import (
             CommunityData,
             ContextData,
             ObjectIdentity,
             ObjectType,
             SnmpEngine,
             UdpTransportTarget,
-        getCmd,
+            get_cmd as _get_cmd,
         )
+        _transport_is_async = True
+    except ImportError:
+        # pysnmp-lextudio 6.2.x
+        from pysnmp.hlapi.asyncio import (
+            CommunityData,
+            ContextData,
+            ObjectIdentity,
+            ObjectType,
+            SnmpEngine,
+            UdpTransportTarget,
+            getCmd as _get_cmd,
+        )
+        _transport_is_async = False
+
+    engine = SnmpEngine()
+
+    # Always release the engine's UDP socket — see _close_engine for why.
+    try:
+        # Same host/port for every OID on this device, so build the transport once.
+        try:
+            if _transport_is_async:
+                transport = await UdpTransportTarget.create((host, port), timeout=5, retries=1)
+            else:
+                transport = UdpTransportTarget((host, port), timeout=5, retries=1)
+        except Exception as exc:
+            logger.debug("SNMP transport setup failed for %s:%s: %s", host, port, exc)
+            return {}
 
         results: dict[str, float] = {}
-    engine = SnmpEngine()
 
         for oid_cfg in oids:
             oid = oid_cfg["oid"]
@@ -62,15 +139,13 @@ def poll_device_sync(
             scale = float(oid_cfg.get("scale", 1.0))
 
             try:
-            error_indication, error_status, error_index, var_binds = next(
-                getCmd(
+                error_indication, error_status, error_index, var_binds = await _get_cmd(
                     engine,
                     CommunityData(community, mpModel=_mp_model(version)),
-                    UdpTransportTarget((host, port), timeout=5, retries=1),
+                    transport,
                     ContextData(),
                     ObjectType(ObjectIdentity(oid)),
                 )
-            )
             except Exception as exc:
                 logger.debug("SNMP GET %s@%s OID %s failed: %s", host, port, oid, exc)
                 continue
@@ -88,7 +163,9 @@ def poll_device_sync(
                 try:
                     results[label] = float(val) * scale
                 except (TypeError, ValueError):
-                # Non-numeric type (e.g. OctetString description) — skip
+                    # Non-numeric type (e.g. OctetString description) — skip.
                     logger.debug("SNMP non-numeric value for %s label=%s: %r", oid, label, val)
 
         return results
+    finally:
+        _close_engine(engine)

plugins/snmp/routes.py

@@ -93,6 +93,75 @@ async def index():
                                  poll_interval=poll_interval)
 
 
+def _device_binds_to_host(d: dict, keys: set[str], host_id: str) -> bool:
+    """Does config device `d` belong on the host identified by `keys`
+    (lowercased {address, name}) / `host_id`?
+
+    A device may declare an **explicit** binding that decouples "which Steward
+    host this belongs to" from "where to poll" (the `host` field):
+      • `host_id`       — the Steward Host UUID (exact match), the explicit link.
+      • `steward_host`  — a friendly bind by Host name or address (case-insensitive).
+    An explicit binding is **exclusive**: a device bound to host A must not also
+    match host B by a coincidental poll-target string. Only when no explicit
+    binding is present do we fall back to the implicit `host`-string match
+    (backward compatible with configs that only set `host`).
+    """
+    explicit_id = str(d.get("host_id", "")).strip()
+    explicit_name = str(d.get("steward_host", "")).strip().lower()
+    if explicit_id or explicit_name:
+        if explicit_id and host_id and explicit_id == host_id:
+            return True
+        return bool(explicit_name and explicit_name in keys)
+    return str(d.get("host", "")).strip().lower() in keys
+
+
+def _devices_for_host(devices_cfg: list, address: str | None, name: str | None,
+                      host_id: str | None = None) -> list[dict]:
+    """SNMP devices that map onto a Steward host's page. Prefers an explicit
+    per-device binding (`host_id` / `steward_host`); otherwise falls back to
+    matching the device's `host` (poll target) against the host's address or
+    name (case-insensitive). See `_device_binds_to_host` for the precedence."""
+    keys = {(address or "").strip().lower(), (name or "").strip().lower()}
+    keys.discard("")
+    hid = (host_id or "").strip()
+    return [
+        d for d in devices_cfg
+        if isinstance(d, dict) and _device_binds_to_host(d, keys, hid)
+    ]
+
+
+@snmp_bp.get("/host/<host_id>")
+@require_role(UserRole.viewer)
+async def host_panel(host_id: str):
+    """Per-host SNMP fragment for the Hosts hub. Surfaces any configured SNMP
+    device that maps to this host (by address or name) with its latest readings.
+    Renders nothing when no device maps, so hosts without SNMP carry no empty card.
+    """
+    from steward.models.hosts import Host
+
+    devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
+    async with current_app.db_sessionmaker() as db:
+        host = (await db.execute(
+            select(Host).where(Host.id == host_id))).scalar_one_or_none()
+        if host is None:
+            return ""
+        matched = _devices_for_host(devices_cfg, host.address, host.name, host.id)
+        if not matched:
+            return ""
+        names = [d.get("name") or d.get("host", "?") for d in matched]
+        latest = await _latest_readings(db, names)
+
+    devices = [{
+        "name": d.get("name") or d.get("host", "?"),
+        "host": d.get("host", ""),
+        "oids": d.get("oids", []),
+        "readings": latest.get(d.get("name") or d.get("host", "?"), {}),
+        "bound": bool(str(d.get("host_id", "")).strip()
+                      or str(d.get("steward_host", "")).strip()),
+    } for d in matched]
+    return await render_template("snmp/host_panel.html", devices=devices)
+
+
 @snmp_bp.get("/widget")
 @require_role(UserRole.viewer)
 async def widget():

plugins/snmp/scheduler.py

@@ -1,6 +1,5 @@
 # plugins/snmp/scheduler.py
 from __future__ import annotations
-import asyncio
 import logging
 from typing import TYPE_CHECKING
 
@@ -27,15 +26,13 @@ def make_poll_task(app: "Quart") -> ScheduledTask:
 
 
 async def _do_poll(app: "Quart") -> None:
-    from .poller import poll_device_sync
+    from .poller import poll_device
     from steward.core.alerts import record_metric
 
     devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", [])
     if not devices:
         return
 
-    loop = asyncio.get_event_loop()
-
     async with app.db_sessionmaker() as session:
         async with session.begin():
             for device in devices:
@@ -52,11 +49,7 @@ async def _do_poll(app: "Quart") -> None:
                     continue
 
                 try:
-                    readings = await loop.run_in_executor(
-                        None,
-                        poll_device_sync,
-                        host, port, community, version, oids,
-                    )
+                    readings = await poll_device(host, port, community, version, oids)
                 except Exception:
                     logger.exception("SNMP poll failed for device %s (%s)", name, host)
                     continue

plugins/snmp/templates/snmp/device.html

@@ -1,9 +1,10 @@
 {# plugins/snmp/templates/snmp/device.html #}
 {% extends "base.html" %}
+{% from "_macros.html" import crumbs %}
 {% block title %}SNMP — {{ device_name }} — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("SNMP", "/plugins/snmp/"), (device_name, "")]) }}{% endblock %}
 {% block content %}
 <div style="display:flex;align-items:center;gap:1rem;margin-bottom:1.5rem;flex-wrap:wrap;">
-  <a href="/plugins/snmp/" style="color:var(--text-muted);font-size:0.85rem;">&#8592; SNMP</a>
   <h1 class="page-title" style="margin:0;">{{ device_name }}</h1>
   <span style="font-size:0.8rem;color:var(--text-dim);">{{ device.host }}</span>
   <div style="margin-left:auto;display:flex;gap:0.5rem;">

plugins/snmp/templates/snmp/host_panel.html

@@ -0,0 +1,51 @@
+{# Per-host SNMP fragment — embedded on the Hosts hub via HTMX. Self-contained.
+   Shows the SNMP device(s) whose address maps to this host + latest readings. #}
+<div class="card">
+  <div style="display:flex;align-items:baseline;justify-content:space-between;margin-bottom:0.6rem;gap:1rem;flex-wrap:wrap;">
+    <h3 class="section-title" style="margin-bottom:0;">SNMP</h3>
+    <a href="/plugins/snmp/" style="font-size:0.8rem;color:var(--text-muted);">All devices →</a>
+  </div>
+
+  {% for device in devices %}
+  <div style="{% if not loop.last %}margin-bottom:1rem;{% endif %}">
+    <div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.5rem;flex-wrap:wrap;">
+      <span style="font-weight:600;font-size:0.92rem;">{{ device.name }}</span>
+      <span style="font-size:0.78rem;color:var(--text-dim);font-family:ui-monospace,monospace;">{{ device.host }}</span>
+      {% if device.bound %}
+      <span title="Explicitly bound to this host (host_id / steward_host)"
+            style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:3px;padding:0.05rem 0.4rem;">bound</span>
+      {% endif %}
+      {% if device.readings %}
+      <span style="font-size:0.74rem;color:var(--green);">&#9679; reachable</span>
+      {% else %}
+      <span style="font-size:0.74rem;color:var(--text-dim);">&#9675; no data yet</span>
+      {% endif %}
+      <a href="/plugins/snmp/device/{{ device.name }}" class="btn btn-ghost btn-sm"
+         style="margin-left:auto;font-size:0.76rem;padding:0.15rem 0.55rem;">History</a>
+    </div>
+
+    {% if device.readings %}
+    <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.5rem;">
+      {% for oid_cfg in device.oids %}
+      {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
+      {% set val = device.readings.get(label) %}
+      <div style="background:var(--bg-elevated);border-radius:5px;padding:0.5rem 0.7rem;border:1px solid var(--border);">
+        <div style="font-size:0.68rem;color:var(--text-dim);margin-bottom:0.2rem;font-family:ui-monospace,monospace;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ label }}</div>
+        {% if val is not none %}
+        <div style="font-size:1rem;font-weight:600;font-variant-numeric:tabular-nums;">
+          {% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}<span style="font-size:0.72rem;color:var(--text-muted);">M</span>
+          {% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}<span style="font-size:0.72rem;color:var(--text-muted);">K</span>
+          {% else %}{{ "%.4g"|format(val) }}{% endif %}
+        </div>
+        {% else %}
+        <div style="font-size:0.85rem;color:var(--text-dim);">—</div>
+        {% endif %}
+      </div>
+      {% endfor %}
+    </div>
+    {% else %}
+    <p style="font-size:0.8rem;color:var(--text-dim);margin:0;">No readings yet — waiting for the next poll.</p>
+    {% endif %}
+  </div>
+  {% endfor %}
+</div>

plugins/traefik/__init__.py

@@ -28,3 +28,8 @@ def get_scheduled_tasks() -> list:
 def get_blueprint():
     from .routes import traefik_bp
     return traefik_bp
+
+
+def get_nav() -> list[dict]:
+    # Surfaces in the sidebar's Infrastructure group (Traefik services view).
+    return [{"label": "Traefik", "href": "/plugins/traefik/"}]

plugins/unifi/__init__.py

@@ -26,3 +26,8 @@ def get_scheduled_tasks() -> list:
 def get_blueprint():
     from .routes import unifi_bp
     return unifi_bp
+
+
+def get_nav() -> list[dict]:
+    # Surfaces in the sidebar's Infrastructure group (UniFi network overview).
+    return [{"label": "UniFi", "href": "/plugins/unifi/"}]

plugins/unifi/scheduler.py

@@ -15,6 +15,25 @@ logger = logging.getLogger(__name__)
 _client = None  # UnifiClient instance, initialised on first scrape
 
 
+async def _drop_client() -> None:
+    """Close and discard the cached client.
+
+    The client holds a long-lived ``httpx.AsyncClient`` (a connection pool over
+    real sockets). Nulling the reference without ``aclose()`` orphans that pool
+    until GC, so every failure tick that re-auths would leak file descriptors —
+    the same class of bug that took the app down via SNMP (Errno 24, "Too many
+    open files"). Close first, then drop. Best-effort: a failed close must not
+    break the poll loop.
+    """
+    global _client
+    if _client is not None:
+        try:
+            await _client.close()
+        except Exception:
+            pass
+        _client = None
+
+
 def make_poll_task(app: "Quart") -> ScheduledTask:
     cfg = app.config["PLUGINS"]["unifi"]
     interval = int(cfg.get("poll_interval_seconds", 60))
@@ -51,7 +70,7 @@ async def _do_poll(app: "Quart") -> None:
             await _client.login()
         except Exception as exc:
             logger.warning("UniFi initial login failed: %s", exc)
-            _client = None
+            await _drop_client()
             return
 
     try:
@@ -60,7 +79,7 @@ async def _do_poll(app: "Quart") -> None:
         devices = await _client.get_devices()
     except Exception as exc:
         logger.warning("UniFi poll failed — will retry next tick: %s", exc)
-        _client = None  # force re-auth on next tick
+        await _drop_client()  # close + force re-auth on next tick
         return
 
     scraped_at = datetime.now(timezone.utc)

plugins/unifi/templates/unifi/widget_clients.html

@@ -1,6 +1,6 @@
 {# unifi/widget_clients.html — dashboard widget: top active clients #}
 {% if not snapshot %}
-<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No client data yet.</div>
+<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No client data yet.</p>
 {% else %}
 
 <div style="display:flex;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">

pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
     "packaging>=24.0",
     "click>=8.0",
     "httpx>=0.27",
+    "cryptography>=42",
 ]
 
 [project.optional-dependencies]

steward/alerts/routes.py

@@ -6,7 +6,6 @@ from sqlalchemy import select
 from steward.auth.middleware import require_role
 from steward.ansible import sources as src_module
 from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertOperator, MaintenanceWindow
-from steward.models.hosts import Host
 from steward.models.metrics import PluginMetric
 from steward.models.users import UserRole
 
@@ -14,15 +13,20 @@ alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
 
 _OPERATORS = [(op.value, op.value) for op in AlertOperator]
 
-# Static catalog: source_module → available metric names
+# Static catalog: source_module → available metric names.
+# Unified monitors emit metrics under their TYPE (icmp/tcp/dns/http), each with
+# is_up (1.0 up / 0.0 down) and response_ms (where the probe times it).
 METRIC_CATALOG: dict[str, list[str]] = {
-    "ping":    ["up", "response_time_ms"],
-    "dns":     ["resolved", "ip_changed"],
+    "icmp":    ["is_up", "response_ms"],
+    "tcp":     ["is_up", "response_ms"],
+    "dns":     ["is_up", "response_ms"],
+    "http":    ["is_up", "response_ms"],
     "traefik": ["request_rate", "error_rate", "latency_p50_ms", "latency_p95_ms",
                 "latency_p99_ms", "response_bytes_rate", "cert_expiry_days"],
     "unifi":   ["is_up", "latency_ms", "total_clients"],
-    "docker":  ["cpu_pct", "mem_pct"],
-    "http":    ["is_up", "response_ms"],
+    # restart_count = cumulative restarts (alert on crash-looping); is_healthy =
+    # 1.0 healthy / 0.0 unhealthy from the container HEALTHCHECK (alert on <1).
+    "docker":  ["cpu_pct", "mem_pct", "restart_count", "is_healthy"],
     "host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
         "disk_used_pct_worst", "load_1m", "load_5m", "load_15m", "uptime_secs",
@@ -38,8 +42,8 @@ _SOURCE_MODULES = list(METRIC_CATALOG.keys())
 async def _get_resources(db, source_module: str) -> list[str]:
     """Return sorted distinct resource names for a source module.
 
-    For ping/dns, supplements with host names so the list is populated
-    even before any metrics have been recorded.
+    For monitor types (icmp/tcp/dns/http), supplements with monitor names so
+    the list is populated even before any metrics have been recorded.
     """
     result = await db.execute(
         select(PluginMetric.resource_name)
@@ -49,9 +53,12 @@ async def _get_resources(db, source_module: str) -> list[str]:
     )
     resources: set[str] = {r for (r,) in result}
 
-    if source_module in ("ping", "dns"):
-        host_result = await db.execute(select(Host.name).order_by(Host.name))
-        for (name,) in host_result:
+    if source_module in ("icmp", "tcp", "dns", "http"):
+        from steward.models.monitors import Monitor
+        mon_result = await db.execute(
+            select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name)
+        )
+        for (name,) in mon_result:
             resources.add(name)
 
     return sorted(resources)

steward/ansible/PLAYBOOK_CONVENTIONS.md

@@ -0,0 +1,39 @@
+# Steward playbook conventions (quick reference)
+
+What Steward reads from a playbook in this subsystem. Full guide:
+`docs/reference/playbook-authoring.md`. Parser:
+`steward/ansible/sources.py` — `discover_playbook_meta()` and
+`discover_playbook_variables()`.
+
+## Magic comments (metadata)
+
+```yaml
+---
+# description: One line on what this playbook does.   # shown on selection
+# steward:category: maintenance                       # grouping badge
+# steward:confirm: true                               # require confirm to run
+- name: …            # used as the description if no `# description:` comment
+  hosts: all         # Steward generates the inventory from the chosen target/group
+```
+
+- `# description:` — first match wins; case-insensitive; falls back to first
+  play `name:`.
+- `# steward:<key>: <value>` — namespaced metadata. Implemented keys:
+  - `category` — free-text grouping label (badge in browse + run form).
+  - `confirm` — `true`/`yes`/`1`/`on` → run form requires a confirmation tick.
+  - `description` — alias for `# description:` (unprefixed wins if both present).
+  - Unknown `# steward:*` keys are ignored (namespace reserved for future use).
+
+## Structural cues (no comments needed)
+
+- **`vars:` scalars** → editable run-time fields (default shown as placeholder;
+  blank = keep default; values sent as highest-precedence extra-vars).
+- **`vars_prompt:`** → run-time fields (required when no default; `private: true`
+  → masked).
+- **Secret naming** — a var whose name contains `password`/`passwd`/`secret`/
+  `token`/`api_key`/`private_key`/`credential` is masked **and not persisted**.
+- **`hosts: all`** — target via the run form's scope; don't hardcode hosts.
+- **Credentials** — never embed them. Steward connects as the managed `steward`
+  account (passwordless sudo; use `become: true`) or one-time bootstrap creds.
+
+Be idempotent — Steward re-runs playbooks (updates, schedules, retries).

steward/ansible/bundled/host_agent/install.yml

@@ -0,0 +1,113 @@
+---
+# description: Install or re-enroll the Steward host agent on an already-provisioned host.
+# steward:category: host-agent
+# Install (or update) the Steward host agent on a target. Mirrors the
+# curl-based install.sh. Pass the per-host registration token as extra-vars:
+#   steward_url=https://steward.example  steward_token=<token from Host Agents>
+# Re-running updates agent.py in place and restarts the service.
+- name: Install Steward host agent
+  hosts: all
+  become: true
+  gather_facts: false
+  # steward_token is a per-host inventory var — not declared here, or a play var
+  # would outrank and shadow it. steward_url arrives as an extra-var.
+  vars:
+    steward_url: ""
+    agent_interval: 30
+  tasks:
+    - name: Require steward_url and steward_token
+      ansible.builtin.assert:
+        that:
+          - steward_url | default('') | length > 0
+          - steward_token | default('') | length > 0
+        fail_msg: "Pass steward_url and steward_token (Steward injects these)."
+
+    - name: Create steward-agent system user
+      ansible.builtin.user:
+        name: steward-agent
+        system: true
+        shell: /usr/sbin/nologin
+        create_home: false
+
+    # Let the agent read the local Docker socket so it can report this host's
+    # containers (the per-host docker collection path). Best-effort: skipped on
+    # hosts without Docker. Restart picks up the new supplementary group.
+    - name: Detect docker group
+      ansible.builtin.command: getent group docker
+      register: _docker_grp
+      changed_when: false
+      failed_when: false
+
+    - name: Add steward-agent to the docker group
+      ansible.builtin.user:
+        name: steward-agent
+        groups: docker
+        append: true
+      when: _docker_grp.rc == 0
+      notify: restart steward-agent
+
+    - name: Create agent directory
+      ansible.builtin.file:
+        path: /usr/local/lib/steward-agent
+        state: directory
+        mode: "0755"
+
+    - name: Download agent.py
+      ansible.builtin.get_url:
+        url: "{{ steward_url }}/plugins/host_agent/agent.py"
+        dest: /usr/local/lib/steward-agent/agent.py
+        mode: "0755"
+        force: true
+      notify: restart steward-agent
+
+    - name: Write agent config
+      ansible.builtin.copy:
+        dest: /etc/steward-agent.conf
+        owner: root
+        group: steward-agent
+        mode: "0640"
+        content: |
+          url = {{ steward_url }}
+          token = {{ steward_token }}
+          interval_seconds = {{ agent_interval }}
+      notify: restart steward-agent
+
+    - name: Install systemd unit
+      ansible.builtin.copy:
+        dest: /etc/systemd/system/steward-agent.service
+        mode: "0644"
+        content: |
+          [Unit]
+          Description=Steward host agent
+          After=network-online.target
+          Wants=network-online.target
+
+          [Service]
+          Type=simple
+          User=steward-agent
+          Environment=STEWARD_AGENT_CONFIG=/etc/steward-agent.conf
+          ExecStart=/usr/bin/env python3 /usr/local/lib/steward-agent/agent.py
+          Restart=on-failure
+          RestartSec=5
+          NoNewPrivileges=true
+          ProtectSystem=strict
+          ProtectHome=true
+          PrivateTmp=true
+
+          [Install]
+          WantedBy=multi-user.target
+      notify: restart steward-agent
+
+    - name: Enable and start the agent
+      ansible.builtin.systemd:
+        name: steward-agent
+        enabled: true
+        state: started
+        daemon_reload: true
+
+  handlers:
+    - name: restart steward-agent
+      ansible.builtin.systemd:
+        name: steward-agent
+        state: restarted
+        daemon_reload: true

steward/ansible/bundled/host_agent/provision.yml

@@ -0,0 +1,169 @@
+---
+# description: Provision a fresh host — create the steward account + managed key + passwordless sudo, then install the agent.
+# steward:category: host-agent
+# First-contact provisioning: bring a fresh host under Steward management, then
+# install the host agent — all in one idempotent run.
+#
+# Run this ONCE per host with bootstrap credentials (an existing login + sudo,
+# typically password auth via the "Provision host" button). It:
+#   1. creates a dedicated `steward` login account,
+#   2. installs Steward's managed public key into its authorized_keys,
+#   3. grants it passwordless sudo,
+#   4. installs/updates the host agent.
+# After this, every future run (agent updates, maintenance) connects as
+# `steward` with the managed key — no operator credentials needed.
+#
+# Extra-vars (injected by the Provision card):
+#   steward_url, steward_token   — agent → Steward auth (per-host)
+#   steward_pubkey               — managed public key to authorize
+#   steward_user (default steward), agent_interval (default 30)
+- name: Provision host for Steward + install agent
+  hosts: all
+  become: true
+  gather_facts: false
+  # steward_token is injected as a per-host inventory var (it differs per host),
+  # so it must NOT be declared here — a play var would outrank the inventory var
+  # and shadow it. The globals below arrive as extra-vars (highest precedence),
+  # which override these defaults.
+  vars:
+    steward_url: ""
+    steward_pubkey: ""
+    steward_user: steward
+    agent_interval: 30
+  tasks:
+    - name: Require provisioning vars
+      ansible.builtin.assert:
+        that:
+          - steward_url | default('') | length > 0
+          - steward_token | default('') | length > 0
+          - steward_pubkey | default('') | length > 0
+        fail_msg: "Pass steward_url, steward_token and steward_pubkey (Steward injects these)."
+
+    # ── Managed login account ────────────────────────────────────────────────
+    - name: Create the steward management account
+      ansible.builtin.user:
+        name: "{{ steward_user }}"
+        shell: /bin/sh
+        create_home: true
+        state: present
+
+    - name: Ensure ~/.ssh exists
+      ansible.builtin.file:
+        path: "/home/{{ steward_user }}/.ssh"
+        state: directory
+        owner: "{{ steward_user }}"
+        group: "{{ steward_user }}"
+        mode: "0700"
+
+    - name: Authorize the managed public key (replace any prior steward-managed key)
+      ansible.builtin.lineinfile:
+        path: "/home/{{ steward_user }}/.ssh/authorized_keys"
+        # Match on the ' steward-managed' comment so a rotated key REPLACES the
+        # old one in place rather than stacking a second authorized key. Any
+        # keys the operator added by hand (different comment) are left untouched.
+        regexp: ' steward-managed$'
+        line: "{{ steward_pubkey }}"
+        create: true
+        owner: "{{ steward_user }}"
+        group: "{{ steward_user }}"
+        mode: "0600"
+
+    - name: Grant passwordless sudo to the steward account
+      ansible.builtin.copy:
+        dest: "/etc/sudoers.d/steward"
+        content: "{{ steward_user }} ALL=(ALL) NOPASSWD:ALL\n"
+        owner: root
+        group: root
+        mode: "0440"
+        validate: "visudo -cf %s"
+
+    # ── Host agent (same as install.yml) ─────────────────────────────────────
+    - name: Create steward-agent system user
+      ansible.builtin.user:
+        name: steward-agent
+        system: true
+        shell: /usr/sbin/nologin
+        create_home: false
+
+    # Let the agent read the local Docker socket so it can report this host's
+    # containers (the per-host docker collection path). Best-effort: skipped on
+    # hosts without Docker. Restart picks up the new supplementary group.
+    - name: Detect docker group
+      ansible.builtin.command: getent group docker
+      register: _docker_grp
+      changed_when: false
+      failed_when: false
+
+    - name: Add steward-agent to the docker group
+      ansible.builtin.user:
+        name: steward-agent
+        groups: docker
+        append: true
+      when: _docker_grp.rc == 0
+      notify: restart steward-agent
+
+    - name: Create agent directory
+      ansible.builtin.file:
+        path: /usr/local/lib/steward-agent
+        state: directory
+        mode: "0755"
+
+    - name: Download agent.py
+      ansible.builtin.get_url:
+        url: "{{ steward_url }}/plugins/host_agent/agent.py"
+        dest: /usr/local/lib/steward-agent/agent.py
+        mode: "0755"
+        force: true
+      notify: restart steward-agent
+
+    - name: Write agent config
+      ansible.builtin.copy:
+        dest: /etc/steward-agent.conf
+        owner: root
+        group: steward-agent
+        mode: "0640"
+        content: |
+          url = {{ steward_url }}
+          token = {{ steward_token }}
+          interval_seconds = {{ agent_interval }}
+      notify: restart steward-agent
+
+    - name: Install systemd unit
+      ansible.builtin.copy:
+        dest: /etc/systemd/system/steward-agent.service
+        mode: "0644"
+        content: |
+          [Unit]
+          Description=Steward host agent
+          After=network-online.target
+          Wants=network-online.target
+
+          [Service]
+          Type=simple
+          User=steward-agent
+          Environment=STEWARD_AGENT_CONFIG=/etc/steward-agent.conf
+          ExecStart=/usr/bin/env python3 /usr/local/lib/steward-agent/agent.py
+          Restart=on-failure
+          RestartSec=5
+          NoNewPrivileges=true
+          ProtectSystem=strict
+          ProtectHome=true
+          PrivateTmp=true
+
+          [Install]
+          WantedBy=multi-user.target
+      notify: restart steward-agent
+
+    - name: Enable and start the agent
+      ansible.builtin.systemd:
+        name: steward-agent
+        enabled: true
+        state: started
+        daemon_reload: true
+
+  handlers:
+    - name: restart steward-agent
+      ansible.builtin.systemd:
+        name: steward-agent
+        state: restarted
+        daemon_reload: true

steward/ansible/bundled/host_agent/update.yml

@@ -0,0 +1,49 @@
+---
+# description: Update the Steward host agent (refresh agent.py + restart) on hosts already running it.
+# steward:category: host-agent
+# Update an already-installed Steward host agent: refresh agent.py and restart.
+#
+# Deliberately does NOT touch the registration token or /etc/steward-agent.conf
+# — the host already has a working token, so an update leaves identity alone and
+# only swaps the agent binary. Connects as the managed `steward` account (set
+# globally as the SSH user); no bootstrap password or token injection needed.
+#
+# For a fresh host with no agent, use provision.yml (new host) or install.yml
+# (already-provisioned host) instead — those mint and install the token.
+- name: Update Steward host agent
+  hosts: all
+  become: true
+  gather_facts: false
+  vars:
+    steward_url: ""
+  tasks:
+    - name: Require steward_url
+      ansible.builtin.assert:
+        that:
+          - steward_url | length > 0
+        fail_msg: "Pass steward_url as an extra-var (the Update action sets it)."
+
+    - name: Confirm the agent is already installed
+      ansible.builtin.stat:
+        path: /usr/local/lib/steward-agent/agent.py
+      register: agent_file
+
+    - name: Fail clearly if the agent is missing
+      ansible.builtin.fail:
+        msg: "No agent at /usr/local/lib/steward-agent — provision or install this host first."
+      when: not agent_file.stat.exists
+
+    - name: Refresh agent.py
+      ansible.builtin.get_url:
+        url: "{{ steward_url }}/plugins/host_agent/agent.py"
+        dest: /usr/local/lib/steward-agent/agent.py
+        mode: "0755"
+        force: true
+      notify: restart steward-agent
+
+  handlers:
+    - name: restart steward-agent
+      ansible.builtin.systemd:
+        name: steward-agent
+        state: restarted
+        daemon_reload: true

steward/ansible/bundled/maintenance/docker_prune.yml

@@ -0,0 +1,64 @@
+---
+# description: Reclaim disk on Docker / Swarm nodes — prune stopped containers, unused images, or a full system prune.
+# steward:category: maintenance
+# steward:confirm: true
+# Reclaim disk on Docker / Docker Swarm nodes by removing unused data.
+# `prune_target` selects the scope (default `system` preserves the original
+# behavior for existing manual/scheduled callers that don't set it):
+#   prune_target=containers   remove stopped containers only (docker container prune)
+#   prune_target=images       remove unused images (docker image prune;
+#                             + prune_all_images=true → -a, i.e. ALL unused, not just dangling)
+#   prune_target=system       docker system prune (dangling images, stopped
+#                             containers, unused networks, build cache). Widen with:
+#                               prune_all_images=true   also ALL unused images
+#                               prune_volumes=true       also unused named volumes (data loss risk)
+- name: Docker prune
+  hosts: all
+  gather_facts: false
+  become: true
+  vars:
+    prune_target: system
+    prune_all_images: false
+    prune_volumes: false
+  tasks:
+    - name: Validate prune_target
+      ansible.builtin.assert:
+        that: prune_target in ['containers', 'images', 'system']
+        fail_msg: "prune_target must be one of: containers, images, system (got '{{ prune_target }}')"
+        quiet: true
+
+    # ── Stopped containers only ───────────────────────────────────────────────
+    - name: Prune stopped containers
+      ansible.builtin.command:
+        argv: ['docker', 'container', 'prune', '-f']
+      register: container_prune
+      changed_when: "'Total reclaimed space: 0B' not in container_prune.stdout"
+      when: prune_target == 'containers'
+
+    # ── Unused images (dangling, or all unused with -a) ───────────────────────
+    - name: Prune unused images
+      ansible.builtin.command:
+        argv: >-
+          {{ ['docker', 'image', 'prune', '-f']
+             + (['-a'] if prune_all_images | bool else []) }}
+      register: image_prune
+      changed_when: "'Total reclaimed space: 0B' not in image_prune.stdout"
+      when: prune_target == 'images'
+
+    # ── Full system prune (default) ───────────────────────────────────────────
+    - name: Run docker system prune
+      ansible.builtin.command:
+        argv: >-
+          {{ ['docker', 'system', 'prune', '-f']
+             + (['-a'] if prune_all_images | bool else [])
+             + (['--volumes'] if prune_volumes | bool else []) }}
+      register: system_prune
+      changed_when: "'Total reclaimed space: 0B' not in system_prune.stdout"
+      when: prune_target == 'system'
+
+    - name: Report reclaimed space
+      ansible.builtin.debug:
+        msg: >-
+          {{ ((container_prune.stdout_lines | default([]))
+              + (image_prune.stdout_lines | default([]))
+              + (system_prune.stdout_lines | default([]))) | select | list }}

steward/ansible/bundled/maintenance/system_update.yml

@@ -0,0 +1,82 @@
+---
+# description: Update all OS packages; optionally restart affected services and reboot if one is pending.
+# steward:category: maintenance
+# steward:confirm: true
+#
+# Cross-distro (apt / dnf-yum). Flags (set in the run form):
+#   restart_services=true   after the upgrade, restart every service that needs
+#                           it (Debian: needrestart -r a; installed if missing).
+#   reboot_if_required=true reboot the host only if the update flagged a pending
+#                           reboot (Debian: /var/run/reboot-required;
+#                           RHEL: needs-restarting -r). Never reboots otherwise.
+- name: System package update
+  hosts: all
+  become: true
+  gather_facts: true
+  vars:
+    restart_services: false
+    reboot_if_required: false
+  tasks:
+    # ── Debian / Ubuntu ──────────────────────────────────────────────────────
+    - name: Upgrade all packages (apt)
+      ansible.builtin.apt:
+        update_cache: true
+        upgrade: dist
+        autoremove: true
+      when: ansible_pkg_mgr == "apt"
+
+    - name: Ensure needrestart is present (apt, for service restarts)
+      ansible.builtin.apt:
+        name: needrestart
+        state: present
+      when:
+        - restart_services | bool
+        - ansible_pkg_mgr == "apt"
+
+    - name: Restart services that need it (apt / needrestart)
+      ansible.builtin.command: needrestart -r a
+      register: needrestart_run
+      changed_when: needrestart_run.rc == 0
+      failed_when: false
+      when:
+        - restart_services | bool
+        - ansible_pkg_mgr == "apt"
+
+    # ── RHEL / Fedora ─────────────────────────────────────────────────────────
+    - name: Upgrade all packages (dnf/yum)
+      ansible.builtin.dnf:
+        name: "*"
+        state: latest
+      when: ansible_pkg_mgr in ("dnf", "yum")
+
+    - name: Restart services that need it (dnf / needs-restarting)
+      ansible.builtin.shell: needs-restarting -s | xargs -r -n1 systemctl try-restart
+      register: rhel_restart
+      changed_when: rhel_restart.rc == 0
+      failed_when: false
+      when:
+        - restart_services | bool
+        - ansible_pkg_mgr in ("dnf", "yum")
+
+    # ── Reboot only if one is pending ─────────────────────────────────────────
+    - name: Check pending reboot (Debian)
+      ansible.builtin.stat:
+        path: /var/run/reboot-required
+      register: deb_reboot
+      when: ansible_pkg_mgr == "apt"
+
+    - name: Check pending reboot (RHEL)
+      ansible.builtin.command: needs-restarting -r
+      register: rhel_reboot
+      changed_when: false
+      failed_when: false
+      when: ansible_pkg_mgr in ("dnf", "yum")
+
+    - name: Reboot if required
+      ansible.builtin.reboot:
+        msg: "Reboot triggered by Steward system_update"
+        reboot_timeout: 900
+      when:
+        - reboot_if_required | bool
+        - (ansible_pkg_mgr == "apt" and deb_reboot.stat.exists)
+          or (ansible_pkg_mgr in ("dnf", "yum") and (rhel_reboot.rc | default(0)) != 0)

steward/ansible/executor.py

@@ -4,6 +4,7 @@ import asyncio
 import json
 import logging
 import os
+import re
 import shutil
 import tempfile
 import time
@@ -12,14 +13,26 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from steward.core.crypto import is_encrypted as _crypto_is_encrypted
+
 if TYPE_CHECKING:
     from quart import Quart
 
 logger = logging.getLogger(__name__)
 
-_OUTPUT_CAP_BYTES = 1024 * 1024  # 1 MB DB cap
+_OUTPUT_CAP_BYTES = 1024 * 1024  # 1 MB DB cap (full output also goes to a log artifact)
 _FLUSH_LINES = 50
 _FLUSH_SECS = 5.0
+_MEM_LINES_CAP = 5000   # bounded per-run in-memory replay buffer (SSE late-joiners)
+_MEM_TTL_SECS = 300     # drop in-memory state this long after a run completes
+_FAILURE_CAP = 50       # max failed-task lines captured into structured results
+
+# Full run logs persist here (survive restart + the 1 MB DB cap). Override via env.
+ARTIFACT_DIR = os.environ.get("STEWARD_ANSIBLE_LOG_DIR", "/data/ansible/runs")
+
+
+def artifact_path(run_id: str) -> str:
+    return os.path.join(ARTIFACT_DIR, f"{run_id}.log")
 
 
 @dataclass
@@ -27,11 +40,19 @@ class _Done:
     status: str
 
 
-# Per-run broadcast state (lives until process restarts)
-_run_lines: dict[str, list[str]] = {}          # all output lines received so far
+class _Aborted(Exception):
+    """Raised internally when a queued run is cancelled before it launches."""
+
+
+# Per-run broadcast state (lives until shortly after completion, then GC'd)
+_run_lines: dict[str, list[str]] = {}          # bounded tail of output lines
 _run_done: dict[str, _Done] = {}               # set when run completes
 _run_listeners: dict[str, list[asyncio.Queue]] = {}  # active SSE client queues
 
+# Running subprocesses by run_id (for cancellation) + run_ids an operator cancelled.
+_run_procs: dict[str, asyncio.subprocess.Process] = {}
+_cancelled: set[str] = set()
+
 
 def register_listener(run_id: str) -> asyncio.Queue:
     """Create a listener queue for a run. Replays existing lines immediately."""
@@ -52,15 +73,31 @@ def deregister_listener(run_id: str, q: asyncio.Queue) -> None:
         listeners.remove(q)
 
 
+def _schedule_cleanup(run_id: str) -> None:
+    """Drop a finished run's in-memory state after a grace window."""
+    async def _cleanup() -> None:
+        await asyncio.sleep(_MEM_TTL_SECS)
+        _run_lines.pop(run_id, None)
+        _run_done.pop(run_id, None)
+    try:
+        asyncio.create_task(_cleanup())
+    except RuntimeError:
+        pass  # no running loop (e.g. tests) — fine to skip
+
+
 def _broadcast(run_id: str, item: str | _Done) -> None:
     if isinstance(item, str):
-        _run_lines.setdefault(run_id, []).append(item)
+        buf = _run_lines.setdefault(run_id, [])
+        buf.append(item)
+        if len(buf) > _MEM_LINES_CAP:  # keep only the most recent lines in memory
+            del buf[: len(buf) - _MEM_LINES_CAP]
     else:
         _run_done[run_id] = item
     for q in _run_listeners.get(run_id, []):
         q.put_nowait(item)
     if isinstance(item, _Done):
         _run_listeners.pop(run_id, None)
+        _schedule_cleanup(run_id)
 
 
 def build_ansible_command(
@@ -98,8 +135,12 @@ def build_credentials(creds: dict | None, tmpdir: str) -> tuple[list[str], list[
     files: list[tuple[str, str]] = []
     creds = creds or {}
 
+    from steward.core.crypto import is_encrypted
     key = creds.get("ssh_private_key")
-    if key:
+    # A still-encrypted value means decryption failed (app secret key changed) —
+    # writing the ciphertext as a key file just yields a cryptic libcrypto error,
+    # so skip it; start_run surfaces a clear message instead.
+    if key and not is_encrypted(key):
         path = os.path.join(tmpdir, "id_key")
         files.append((path, key if key.endswith("\n") else key + "\n"))
         args += ["--private-key", path]
@@ -120,6 +161,51 @@ def build_credentials(creds: dict | None, tmpdir: str) -> tuple[list[str], list[
     return args, files
 
 
+def build_bootstrap(connection: dict | None, tmpdir: str) -> tuple[list[str], list[tuple[str, str]]]:
+    """Per-run connection override for first-contact provisioning.
+
+    Pure: returns (extra argv, files to write). The bootstrap user/password are
+    written to a temp vars file (``-e @file``) — never argv, never the DB — so
+    they don't leak via the process list or persist on the AnsibleRun row.
+    A bare connection password doubles as the become password unless one is
+    given explicitly. connection keys: user, password, become_password.
+    """
+    args: list[str] = []
+    files: list[tuple[str, str]] = []
+    connection = connection or {}
+    user = (connection.get("user") or "").strip()
+    password = connection.get("password") or ""
+    if not user and not password:
+        return args, files
+
+    lines: dict[str, str] = {}
+    if user:
+        lines["ansible_user"] = user
+    if password:
+        lines["ansible_password"] = password
+        lines["ansible_become_password"] = connection.get("become_password") or password
+    # JSON values are valid YAML and keep arbitrary characters safe.
+    content = "".join(f"{k}: {json.dumps(v)}\n" for k, v in lines.items())
+    path = os.path.join(tmpdir, "bootstrap.yml")
+    files.append((path, content))
+    args += ["-e", "@" + path]
+    return args, files
+
+
+def build_extra_vars_file(merged: dict | None, tmpdir: str) -> tuple[list[str], list[tuple[str, str]]]:
+    """Run-time variables → a JSON extra-vars file (``-e @file``).
+
+    Pure: returns (argv, files). JSON is valid YAML to Ansible and, unlike
+    ``-e key=value`` strings (which Ansible shlex-splits on whitespace), keeps
+    values with spaces/quotes intact. Used for both operator-entered playbook
+    variables and the unpersisted secret subset (merged by the caller).
+    """
+    if not merged:
+        return [], []
+    path = os.path.join(tmpdir, "extravars.json")
+    return ["-e", "@" + path], [(path, json.dumps(merged))]
+
+
 def ansible_env(creds: dict | None, base_env) -> dict:
     """Return a copy of base_env with ANSIBLE_HOST_KEY_CHECKING set from creds."""
     env = dict(base_env)
@@ -127,6 +213,86 @@ def ansible_env(creds: dict | None, base_env) -> dict:
     return env
 
 
+# ── Structured results (parsed from the streamed output) ──────────────────────
+
+_RECAP_RE = re.compile(
+    r"^(?P<host>\S+)\s*:\s*ok=(?P<ok>\d+)\s+changed=(?P<changed>\d+)\s+"
+    r"unreachable=(?P<unreachable>\d+)\s+failed=(?P<failed>\d+)"
+    r"(?:\s+skipped=(?P<skipped>\d+))?"
+)
+
+
+def parse_recap(lines: list[str]) -> dict[str, dict[str, int]]:
+    """Parse Ansible's PLAY RECAP into {host: {ok,changed,unreachable,failed,skipped}}.
+
+    Streaming-friendly: we keep the default stdout callback (live output) and
+    derive the per-host summary from the recap rather than swapping to the json
+    callback (which would buffer everything to the end and kill live streaming).
+    """
+    hosts: dict[str, dict[str, int]] = {}
+    in_recap = False
+    for line in lines:
+        if "PLAY RECAP" in line:
+            in_recap = True
+            continue
+        if not in_recap:
+            continue
+        m = _RECAP_RE.match(line.strip())
+        if m:
+            hosts[m.group("host")] = {
+                "ok": int(m.group("ok")),
+                "changed": int(m.group("changed")),
+                "unreachable": int(m.group("unreachable")),
+                "failed": int(m.group("failed")),
+                "skipped": int(m.group("skipped") or 0),
+            }
+    return hosts
+
+
+# ── Cancellation ──────────────────────────────────────────────────────────────
+
+async def _set_status(app: "Quart", run_id: str, status) -> None:
+    from sqlalchemy import update
+    from steward.models.ansible import AnsibleRun
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            await session.execute(
+                update(AnsibleRun).where(AnsibleRun.id == run_id).values(status=status)
+            )
+
+
+async def _terminate_then_kill(proc: asyncio.subprocess.Process, grace: float = 10.0) -> None:
+    try:
+        proc.terminate()
+    except ProcessLookupError:
+        return
+    try:
+        await asyncio.wait_for(proc.wait(), timeout=grace)
+    except asyncio.TimeoutError:
+        try:
+            proc.kill()
+        except ProcessLookupError:
+            pass
+
+
+def cancel_run(run_id: str) -> bool:
+    """Request cancellation of a run. Terminates the process if it's running;
+    a queued run is flagged so start_run aborts before launch. Returns False
+    only when the run isn't tracked in this process (already finished)."""
+    if run_id not in _run_procs and run_id not in _cancelled:
+        # Not obviously in-flight. Still flag it (covers a just-queued run whose
+        # proc hasn't been registered yet); caller should gate on run status.
+        _cancelled.add(run_id)
+        return run_id in _run_procs or True
+    _cancelled.add(run_id)
+    proc = _run_procs.get(run_id)
+    if proc is not None:
+        asyncio.create_task(_terminate_then_kill(proc))
+    return True
+
+
+# ── Run execution ─────────────────────────────────────────────────────────────
+
 async def start_run(
     app: "Quart",
     run_id: str,
@@ -135,29 +301,42 @@ async def start_run(
     source_path: str,
     params: dict | None = None,
     inventory_content: str | None = None,
+    connection: dict | None = None,
+    secret_vars: dict | None = None,
 ) -> None:
     """Execute ansible-playbook as a subprocess and update the DB run row.
 
-    If inventory_content is given (host-targeted runs), it's written to a temp
-    file and used as the inventory instead of inventory_path.
-    """
-    _run_lines[run_id] = []
+    Respects a global concurrency cap (app._ansible_semaphore): if no slot is
+    free the run shows as 'queued' until one frees. Supports cancellation, a
+    persistent full-log artifact, and a parsed per-host result summary.
 
+    connection is an optional per-run SSH override (user/password) for
+    first-contact provisioning. It is applied to the subprocess only — it is
+    never persisted on the AnsibleRun row, never placed on argv, and not logged.
+    secret_vars are run-time playbook variables flagged sensitive; like
+    connection they reach the subprocess (merged into the extra-vars file) but
+    are never persisted.
+    """
+    from steward.models.ansible import AnsibleRunStatus
+
+    _run_lines[run_id] = []
     db_output = ""
     truncated = False
     lines_since_flush = 0
     last_flush_at = time.monotonic()
+    failures: list[str] = []
 
-    async def _flush(final_status: str | None = None) -> None:
+    async def _flush(final_status: str | None = None, results: dict | None = None) -> None:
         nonlocal lines_since_flush, last_flush_at
         from sqlalchemy import update
-        from steward.models.ansible import AnsibleRun, AnsibleRunStatus
+        from steward.models.ansible import AnsibleRun
 
         values: dict = {"output": db_output}
+        if results is not None:
+            values["results"] = results
         if final_status is not None:
             values["status"] = AnsibleRunStatus(final_status)
             values["finished_at"] = datetime.now(timezone.utc)
-
         async with app.db_sessionmaker() as session:
             async with session.begin():
                 await session.execute(
@@ -166,44 +345,104 @@ async def start_run(
         lines_since_flush = 0
         last_flush_at = time.monotonic()
 
+    # ── Concurrency gate (semaphore lazily bound to the running loop so the
+    # cap is honoured in prod's single loop without breaking multi-loop tests).
+    loop = asyncio.get_running_loop()
+    sem = getattr(app, "_ansible_semaphore", None)
+    if sem is None or getattr(app, "_ansible_sem_loop", None) is not loop:
+        cap = int((app.config.get("ANSIBLE") or {}).get("max_concurrent_runs", 3) or 3)
+        sem = asyncio.Semaphore(max(1, cap))
+        app._ansible_semaphore = sem
+        app._ansible_sem_loop = loop
+    queued = sem.locked()
+    if queued:
+        await _set_status(app, run_id, AnsibleRunStatus.queued)
+        _broadcast(run_id, "[queued — waiting for an Ansible run slot]")
+    await sem.acquire()
+
+    final_status = "failed"
+    logf = None
+    try:
+        if run_id in _cancelled:  # cancelled while queued — abort before launching
+            raise _Aborted
+        if queued:
+            await _set_status(app, run_id, AnsibleRunStatus.running)
+
         cwd = source_path if Path(source_path).exists() else None
         creds = app.config.get("ANSIBLE", {})
+        try:
+            os.makedirs(ARTIFACT_DIR, exist_ok=True)
+            logf = open(artifact_path(run_id), "w", encoding="utf-8")
+        except OSError:
+            logf = None  # artifact is best-effort; the DB still holds capped output
 
-    # Temp dir holds the ephemeral inventory (if any) + credential files; removed in finally.
+        # Temp dir holds the ephemeral inventory (if any) + credential files.
         tmpdir = tempfile.mkdtemp(prefix="steward-ansible-")
         os.chmod(tmpdir, 0o700)
         try:
             if inventory_content:
-            inv_target = os.path.join(tmpdir, "inventory")
+                # .yml so Ansible's yaml inventory plugin reliably claims it
+                # (the plugin's verify_file checks the extension).
+                inv_target = os.path.join(tmpdir, "inventory.yml")
                 with open(inv_target, "w", encoding="utf-8") as invf:
                     invf.write(inventory_content)
             else:
                 inv_target = inventory_path
             cmd = build_ansible_command(playbook_path, inv_target, params)
 
-        cred_args, cred_files = build_credentials(creds, tmpdir)
-        for cred_path, content in cred_files:
+            # First-contact provisioning supplies a password and a bootstrap
+            # user; the managed key isn't on the host yet, so don't offer it
+            # (avoids a failed key attempt before the password fallback).
+            conn = connection or {}
+            effective_creds = dict(creds)
+            if conn.get("password"):
+                effective_creds.pop("ssh_private_key", None)
+            elif _crypto_is_encrypted(effective_creds.get("ssh_private_key") or ""):
+                _broadcast(run_id,
+                           "[run error] The managed SSH key could not be decrypted "
+                           "(the app secret key changed). Regenerate it in "
+                           "Settings → Ansible and re-provision the host(s).")
+
+            cred_args, cred_files = build_credentials(effective_creds, tmpdir)
+            boot_args, boot_files = build_bootstrap(conn, tmpdir)
+            # Operator-entered run-time vars (persisted) + secret vars (not).
+            merged_vars = {**((params or {}).get("extra_vars_map") or {}), **(secret_vars or {})}
+            ev_args, ev_files = build_extra_vars_file(merged_vars, tmpdir)
+            for cred_path, content in cred_files + boot_files + ev_files:
                 with open(cred_path, "w", encoding="utf-8") as cf:
                     cf.write(content)
                 os.chmod(cred_path, 0o600)
 
+            # Steady-state floor: connect as the managed account unless a target
+            # var or the bootstrap override (extra-vars, higher precedence) wins.
+            user_args: list[str] = []
+            if not conn.get("user"):
+                ssh_user = (creds.get("ssh_user") or "").strip()
+                if ssh_user:
+                    user_args += ["--user", ssh_user]
+
             proc = await asyncio.create_subprocess_exec(
-            *cmd, *cred_args,
+                *cmd, *cred_args, *boot_args, *user_args, *ev_args,
                 stdout=asyncio.subprocess.PIPE,
                 stderr=asyncio.subprocess.STDOUT,
                 cwd=cwd,
                 env=ansible_env(creds, os.environ),
             )
+            _run_procs[run_id] = proc
             assert proc.stdout is not None
 
             async for raw_line in proc.stdout:
                 line = raw_line.decode(errors="replace").rstrip("\n\r")
                 _broadcast(run_id, line)
+                if logf is not None:
+                    logf.write(line + "\n")
+                if (line.startswith("fatal:") or "FAILED!" in line) and len(failures) < _FAILURE_CAP:
+                    failures.append(line[:500])
 
                 if not truncated:
                     candidate = (db_output + "\n" + line) if db_output else line
                     if len(candidate.encode()) > _OUTPUT_CAP_BYTES:
-                    db_output += "\n[output truncated]"
+                        db_output += "\n[output truncated — download the full log]"
                         truncated = True
                     else:
                         db_output = candidate
@@ -214,13 +453,43 @@ async def start_run(
                     await _flush()
 
             await proc.wait()
-        final_status = "success" if proc.returncode == 0 else "failed"
-
-    except Exception:
-        logger.exception("Ansible run %s failed with exception", run_id)
+            recap = parse_recap(_run_lines.get(run_id, []))
+            if run_id in _cancelled:
+                final_status = "cancelled"
+            elif proc.returncode != 0:
                 final_status = "failed"
+            elif not recap:
+                # ansible-playbook exits 0 on "no hosts matched" / empty inventory
+                # — nothing actually ran, so don't report it as a success.
+                final_status = "failed"
+                if len(failures) < _FAILURE_CAP:
+                    failures.append(
+                        "No hosts matched — nothing executed "
+                        "(check the host's Ansible target and inventory).")
+            else:
+                final_status = "success"
         finally:
             shutil.rmtree(tmpdir, ignore_errors=True)
 
-    await _flush(final_status)
+    except _Aborted:
+        final_status = "cancelled"
+    except Exception as exc:
+        logger.exception("Ansible run %s failed with exception", run_id)
+        final_status = "cancelled" if run_id in _cancelled else "failed"
+        # Surface WHY in the run output instead of an empty "failed" (e.g. ENOSPC
+        # when the temp dir can't be created — the run never reaches Ansible).
+        msg = f"[run error] {type(exc).__name__}: {exc}"
+        _broadcast(run_id, msg)
+        if len(failures) < _FAILURE_CAP:
+            failures.append(msg[:500])
+        db_output = (db_output + "\n" + msg) if db_output else msg
+    finally:
+        if logf is not None:
+            logf.close()
+        _run_procs.pop(run_id, None)
+        _cancelled.discard(run_id)
+        sem.release()
+
+    results = {"hosts": parse_recap(_run_lines.get(run_id, [])), "failures": failures}
+    await _flush(final_status, results=results)
     _broadcast(run_id, _Done(status=final_status))

steward/ansible/inventory_gen.py

@@ -37,6 +37,33 @@ def generate_inventory(targets: list) -> dict:
     return inv
 
 
+def inventory_to_yaml(inv: dict) -> str:
+    """Convert the ``--list`` dict from generate_inventory into a STATIC YAML
+    inventory string.
+
+    The ``--list`` shape (``all.hosts`` is a *list*, vars under ``_meta``) is only
+    valid as the stdout of an executable dynamic-inventory script. When written to
+    a plain file and passed via ``-i``, Ansible's yaml plugin requires
+    ``all.hosts`` to be a *dict* keyed by hostname — otherwise it fails to parse
+    and silently falls back to implicit localhost. This emits that valid form.
+    """
+    import yaml
+    hostvars = (inv.get("_meta") or {}).get("hostvars") or {}
+    all_hosts = (inv.get("all") or {}).get("hosts") or []
+    root: dict = {"hosts": {h: (hostvars.get(h) or {}) for h in all_hosts}}
+    children: dict = {}
+    for key, val in inv.items():
+        if key in ("all", "_meta") or not isinstance(val, dict):
+            continue
+        grp: dict = {"hosts": {h: {} for h in (val.get("hosts") or [])}}
+        if val.get("vars"):
+            grp["vars"] = val["vars"]
+        children[key] = grp
+    if children:
+        root["children"] = children
+    return yaml.safe_dump({"all": root}, default_flow_style=False, sort_keys=True)
+
+
 async def fetch_scope_targets(db, scope: str) -> list:
     """Query AnsibleTarget objects for a given inventory scope string.
 

steward/ansible/routes.py

@@ -1,20 +1,28 @@
 # steward/ansible/routes.py
 from __future__ import annotations
-import asyncio
-import uuid
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 
 from quart import (
-    Blueprint, Response, current_app, render_template,
-    request, session,
+    Blueprint, Response, current_app, redirect, render_template,
+    request, session, url_for,
 )
 from sqlalchemy import select
 
 from steward.ansible import executor, sources as src_module
 from steward.auth.middleware import require_role
 from steward.models.ansible import AnsibleRun, AnsibleRunStatus
+from steward.models.ansible_schedule import AnsibleSchedule
 from steward.models.users import User, UserRole
 
+INTERVAL_PRESETS = [
+    (300, "Every 5 minutes"),
+    (3600, "Hourly"),
+    (21600, "Every 6 hours"),
+    (43200, "Every 12 hours"),
+    (86400, "Daily"),
+    (604800, "Weekly"),
+]
+
 ansible_bp = Blueprint("ansible", __name__, url_prefix="/ansible")
 
 
@@ -42,12 +50,22 @@ async def browse():
     all_sources = _get_sources()
     source_data = []
     for source in all_sources:
-        playbooks = src_module.discover_playbooks(source["path"])
+        playbooks = []
+        for path in src_module.discover_playbooks(source["path"]):
+            contents = src_module.read_playbook(source["path"], path)
+            meta = (src_module.discover_playbook_meta(contents) if contents is not None
+                    else {"description": "", "category": ""})
+            playbooks.append({
+                "path": path,
+                "category": meta["category"],
+                "description": meta["description"],
+            })
         inventories = src_module.discover_inventories(source["path"])
         source_data.append({
             "source": source,
             "playbooks": playbooks,
             "inventories": inventories,
+            "editable": src_module.is_editable_source(source),
         })
 
     async with current_app.db_sessionmaker() as db:
@@ -82,14 +100,130 @@ async def view_playbook(source_name: str, playbook_path: str):
         view_source=source_name,
         view_path=playbook_path,
         view_contents=contents,
+        view_editable=src_module.is_editable_source(source),
+    )
+
+
+def _parse_run_params(form) -> tuple[dict | None, dict | None, str | None]:
+    """Build executor params + the unpersisted secret-vars dict from form fields.
+
+    Returns (params, secret_vars, error). Discovered variable fields are named
+    ``var__<name>``; a sibling hidden ``secret__<name>`` marks the sensitive
+    ones, which go into secret_vars (never persisted) instead of params. A
+    free-form ``extra_vars`` textarea (key=value per line) is also folded in.
+    All run-time values flow through a JSON extra-vars file downstream, so
+    values may contain spaces/quotes safely.
+    """
+    extra_vars_map: dict[str, str] = {}
+    secret_vars: dict[str, str] = {}
+
+    for key in set(form.keys()):
+        if not key.startswith("var__"):
+            continue
+        name = key[len("var__"):]
+        val = (form.get(key) or "").strip()
+        if not val:
+            continue  # untouched → fall through to inventory/play default
+        if f"secret__{name}" in form:
+            secret_vars[name] = val
+        else:
+            extra_vars_map[name] = val
+
+    for line in (form.get("extra_vars", "") or "").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        if "=" not in line:
+            return None, None, f"Invalid extra var (expected key=value): {line!r}"
+        k, v = line.split("=", 1)
+        extra_vars_map[k.strip()] = v
+
+    params: dict = {}
+    if extra_vars_map:
+        params["extra_vars_map"] = extra_vars_map
+    limit = (form.get("limit", "") or "").strip()
+    if limit:
+        params["limit"] = limit
+    tags = (form.get("tags", "") or "").strip()
+    if tags:
+        params["tags"] = tags
+    if "check" in form:
+        params["check"] = True
+    return (params or None), (secret_vars or None), None
+
+
+@ansible_bp.get("/playbook-options")
+@require_role(UserRole.operator)
+async def playbook_options():
+    """HTMX fragment: <option>s of playbooks in a source, to populate a playbook
+    dropdown when a source is chosen. `selected` pre-selects one (edit forms)."""
+    source_name = (request.args.get("source_name", "") or "").strip()
+    selected = (request.args.get("selected", "") or "").strip()
+    source = next((s for s in _get_sources() if s["name"] == source_name), None)
+    playbooks = src_module.discover_playbooks(source["path"]) if source else []
+    return await render_template(
+        "ansible/_playbook_options.html", playbooks=playbooks, selected=selected)
+
+
+@ansible_bp.get("/playbook-vars")
+@require_role(UserRole.operator)
+async def playbook_vars():
+    """HTMX fragment: fill-in fields for a playbook's declared variables
+    (vars/vars_prompt), loaded when a playbook is chosen from a dropdown."""
+    source_name = (request.args.get("source_name", "") or "").strip()
+    playbook_path = (request.args.get("playbook_path", "") or "").strip()
+    # Schedule form: schedules can't prompt, so secret vars render disabled.
+    schedule = (request.args.get("schedule", "") or "").strip() == "1"
+    source = next((s for s in _get_sources() if s["name"] == source_name), None)
+    variables: list = []
+    meta: dict = {"description": "", "category": "", "confirm": False}
+    if source and playbook_path:
+        contents = src_module.read_playbook(source["path"], playbook_path)
+        if contents is not None:
+            variables = src_module.discover_playbook_variables(contents)
+            meta = src_module.discover_playbook_meta(contents)
+    return await render_template(
+        "ansible/_playbook_vars.html", variables=variables,
+        description=meta["description"], category=meta["category"],
+        confirm=meta["confirm"], schedule=schedule)
+
+
+@ansible_bp.get("/run-form/<source_name>/<path:playbook_path>")
+@require_role(UserRole.operator)
+async def run_form(source_name: str, playbook_path: str):
+    """HTMX fragment: the run form for one playbook, with a field per declared
+    variable (vars/vars_prompt) discovered from the playbook itself."""
+    from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup
+
+    source = next((s for s in _get_sources() if s["name"] == source_name), None)
+    if source is None:
+        return "Source not found", 404
+    contents = src_module.read_playbook(source["path"], playbook_path)
+    if contents is None:
+        return "Playbook not found", 404
+    variables = src_module.discover_playbook_variables(contents)
+    meta = src_module.discover_playbook_meta(contents)
+    inventories = src_module.discover_inventories(source["path"])
+
+    async with current_app.db_sessionmaker() as db:
+        targets = (await db.execute(
+            select(AnsibleTarget).order_by(AnsibleTarget.name))).scalars().all()
+        groups = (await db.execute(
+            select(AnsibleGroup).order_by(AnsibleGroup.name))).scalars().all()
+
+    return await render_template(
+        "ansible/_run_form.html",
+        source_name=source_name, playbook_path=playbook_path,
+        variables=variables, description=meta["description"], category=meta["category"],
+        confirm=meta["confirm"], targets=targets, groups=groups,
+        inventories=inventories,
     )
 
 
 @ansible_bp.post("/runs")
 @require_role(UserRole.operator)
 async def create_run():
-    import json as _json
-    from steward.ansible.inventory_gen import fetch_scope_targets, generate_inventory
+    from steward.ansible.runner import trigger_run
 
     form = await request.form
     playbook_path = form.get("playbook_path", "").strip()
@@ -99,82 +233,23 @@ async def create_run():
     if not playbook_path:
         return "playbook_path is required", 400
 
-    all_sources = _get_sources()
-    source = next((s for s in all_sources if s["name"] == source_name), None)
-    if source is None:
-        return "Source not found", 404
+    params_or_none, secret_vars, err = _parse_run_params(form)
+    if err:
+        return err, 400
 
-    # Resolve inventory for this scope.
-    inventory_content: str | None = None
-    inventory_path: str | None = None
-    if inventory_scope.startswith("steward:"):
-        async with current_app.db_sessionmaker() as db:
-            targets = await fetch_scope_targets(db, inventory_scope)
-        inventory_content = _json.dumps(generate_inventory(targets))
-    elif inventory_scope.startswith("repo:"):
-        parts = inventory_scope.split(":", 2)
-        inventory_path = parts[2] if len(parts) == 3 else ""
-    else:
-        return "Invalid inventory_scope", 400
-
-    # Optional run parameters (all passed as argv by the executor — no shell).
-    extra_vars: list[str] = []
-    for line in (form.get("extra_vars", "") or "").splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        if "=" not in line:
-            return f"Invalid extra var (expected key=value): {line!r}", 400
-        extra_vars.append(line)
-    limit = (form.get("limit", "") or "").strip()
-    tags = (form.get("tags", "") or "").strip()
-    check = "check" in form
-
-    params: dict = {}
-    if extra_vars:
-        params["extra_vars"] = extra_vars
-    if limit:
-        params["limit"] = limit
-    if tags:
-        params["tags"] = tags
-    if check:
-        params["check"] = True
-    params_or_none = params or None
-
-    run_id = str(uuid.uuid4())
-    now = datetime.now(timezone.utc)
-
-    run = AnsibleRun(
-        id=run_id,
-        playbook_path=playbook_path,
-        inventory_path=inventory_path,
-        inventory_scope=inventory_scope,
-        source_name=source_name,
-        triggered_by=session["user_id"],
-        status=AnsibleRunStatus.running,
-        started_at=now,
-        params=params_or_none,
-    )
-    async with current_app.db_sessionmaker() as db:
-        async with db.begin():
-            db.add(run)
-
-    task = asyncio.create_task(
-        executor.start_run(
+    run, source, err = await trigger_run(
         current_app._get_current_object(),  # type: ignore[attr-defined]
-            run_id,
-            playbook_path,
-            inventory_path or "",
-            source["path"],
-            params_or_none,
-            inventory_content,
-        )
-    )
-    task.add_done_callback(
-        lambda t: t.exception() and current_app.logger.error(
-            "Ansible run %s raised: %s", run_id, t.exception()
-        )
+        source_name=source_name,
+        playbook_path=playbook_path,
+        inventory_scope=inventory_scope,
+        params=params_or_none,
+        secret_vars=secret_vars,
+        triggered_by=session["user_id"],
     )
+    if err == "Source not found":
+        return err, 404
+    if err:
+        return err, 400
 
     return await render_template(
         "ansible/run_started.html",
@@ -193,7 +268,7 @@ async def stream_run(run_id: str):
         return "Not found", 404
 
     async def generate():
-        if run.status != AnsibleRunStatus.running:
+        if run.status not in (AnsibleRunStatus.running, AnsibleRunStatus.queued):
             yield f"event: done\ndata: {run.status.value}\n\n"
             return
 
@@ -233,3 +308,311 @@ async def run_detail(run_id: str):
             triggered_label = res.scalar_one_or_none() or "(deleted user)"
     return await render_template(
         "ansible/run_detail.html", run=run, triggered_label=triggered_label)
+
+
+@ansible_bp.post("/runs/<run_id>/cancel")
+@require_role(UserRole.operator)
+async def cancel_run(run_id: str):
+    async with current_app.db_sessionmaker() as db:
+        run = (await db.execute(
+            select(AnsibleRun).where(AnsibleRun.id == run_id))).scalar_one_or_none()
+    if run is None:
+        return "Not found", 404
+    if run.status in (AnsibleRunStatus.running, AnsibleRunStatus.queued):
+        executor.cancel_run(run_id)
+    return redirect(url_for("ansible.run_detail", run_id=run_id))
+
+
+@ansible_bp.get("/runs/<run_id>/log")
+@require_role(UserRole.viewer)
+async def run_log(run_id: str):
+    """Download the full run log (persistent artifact; falls back to DB output)."""
+    import os
+    from steward.ansible.executor import artifact_path
+
+    async with current_app.db_sessionmaker() as db:
+        run = (await db.execute(
+            select(AnsibleRun).where(AnsibleRun.id == run_id))).scalar_one_or_none()
+    if run is None:
+        return "Not found", 404
+
+    path = artifact_path(run_id)
+    if os.path.exists(path):
+        with open(path, encoding="utf-8", errors="replace") as f:
+            body = f.read()
+    else:
+        body = run.output or ""
+    return Response(
+        body, mimetype="text/plain",
+        headers={"Content-Disposition": f'attachment; filename="ansible-run-{run_id[:8]}.log"'},
+    )
+
+
+# ── Scheduled recurring runs ──────────────────────────────────────────────────
+
+@ansible_bp.get("/schedules")
+@require_role(UserRole.viewer)
+async def schedules():
+    from steward.models.ansible_inventory import AnsibleTarget, AnsibleGroup
+
+    editing_id = request.args.get("edit")
+    async with current_app.db_sessionmaker() as db:
+        scheds = (await db.execute(
+            select(AnsibleSchedule).order_by(AnsibleSchedule.name))).scalars().all()
+        groups = (await db.execute(
+            select(AnsibleGroup).order_by(AnsibleGroup.name))).scalars().all()
+        targets = (await db.execute(
+            select(AnsibleTarget).order_by(AnsibleTarget.name))).scalars().all()
+        run_ids = [s.last_run_id for s in scheds if s.last_run_id]
+        runs = {
+            r.id: r for r in (await db.execute(
+                select(AnsibleRun).where(AnsibleRun.id.in_(run_ids)))).scalars().all()
+        } if run_ids else {}
+
+    source_data = [
+        {"name": s["name"], "playbooks": src_module.discover_playbooks(s["path"])}
+        for s in _get_sources()
+    ]
+
+    rows = []
+    editing = None
+    for s in scheds:
+        rows.append({
+            "s": s,
+            "last_run": runs.get(s.last_run_id) if s.last_run_id else None,
+            "next_run": (s.last_run_at + timedelta(seconds=s.interval_seconds)) if s.last_run_at else None,
+        })
+        if editing_id and s.id == editing_id:
+            editing = s
+
+    # Default the source dropdown to the first source (or the edited schedule's
+    # source), and pre-render that source's playbooks so the playbook dropdown
+    # starts on a real item rather than a placeholder.
+    sel_source = editing.source_name if editing else (source_data[0]["name"] if source_data else "")
+    sel_playbooks = next((sd["playbooks"] for sd in source_data if sd["name"] == sel_source), [])
+
+    # Edit mode: pre-render the selected playbook's declared variables so their
+    # saved values show (fresh loads fetch these via HTMX from /playbook-vars).
+    edit_variables: list = []
+    edit_meta: dict = {"description": "", "category": "", "confirm": False}
+    if editing:
+        src = next((s for s in _get_sources() if s["name"] == editing.source_name), None)
+        if src:
+            contents = src_module.read_playbook(src["path"], editing.playbook_path)
+            if contents is not None:
+                edit_variables = src_module.discover_playbook_variables(contents)
+                edit_meta = src_module.discover_playbook_meta(contents)
+
+    return await render_template(
+        "ansible/schedules.html",
+        rows=rows, groups=groups, targets=targets,
+        source_data=source_data, sel_source=sel_source, sel_playbooks=sel_playbooks,
+        editing=editing, interval_presets=INTERVAL_PRESETS,
+        edit_variables=edit_variables, edit_description=edit_meta["description"],
+        edit_category=edit_meta["category"], edit_confirm=edit_meta["confirm"],
+    )
+
+
+def _schedule_form_fields(form) -> tuple[dict | None, str | None]:
+    """Validate + extract schedule fields from a form. Returns (fields, error)."""
+    name = (form.get("name", "") or "").strip()
+    source_name = (form.get("source_name", "") or "").strip()
+    playbook_path = (form.get("playbook_path", "") or "").strip()
+    inventory_scope = (form.get("inventory_scope", "steward:all") or "steward:all").strip()
+    try:
+        interval = int(form.get("interval_seconds", "0"))
+    except (TypeError, ValueError):
+        interval = 0
+    if not name or not source_name or not playbook_path or interval <= 0:
+        return None, "name, source, playbook, and a positive interval are required"
+    # Scheduled runs can't prompt, so secret vars are intentionally dropped —
+    # automation should rely on global creds / inventory vars, not run-time secrets.
+    params, _secret_vars, err = _parse_run_params(form)
+    if err:
+        return None, err
+    return {
+        "name": name, "source_name": source_name, "playbook_path": playbook_path,
+        "inventory_scope": inventory_scope, "interval_seconds": interval, "params": params,
+    }, None
+
+
+@ansible_bp.post("/schedules")
+@require_role(UserRole.operator)
+async def create_schedule():
+    form = await request.form
+    fields, err = _schedule_form_fields(form)
+    if err:
+        return err, 400
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            db.add(AnsibleSchedule(enabled=True, **fields))
+    return redirect(url_for("ansible.schedules"))
+
+
+@ansible_bp.post("/schedules/<schedule_id>")
+@require_role(UserRole.operator)
+async def update_schedule(schedule_id: str):
+    form = await request.form
+    fields, err = _schedule_form_fields(form)
+    if err:
+        return err, 400
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            sched = await db.get(AnsibleSchedule, schedule_id)
+            if sched is None:
+                return "Not found", 404
+            for k, v in fields.items():
+                setattr(sched, k, v)
+    return redirect(url_for("ansible.schedules"))
+
+
+@ansible_bp.post("/schedules/<schedule_id>/toggle")
+@require_role(UserRole.operator)
+async def toggle_schedule(schedule_id: str):
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            sched = await db.get(AnsibleSchedule, schedule_id)
+            if sched is None:
+                return "Not found", 404
+            sched.enabled = not sched.enabled
+    return redirect(url_for("ansible.schedules"))
+
+
+@ansible_bp.post("/schedules/<schedule_id>/delete")
+@require_role(UserRole.operator)
+async def delete_schedule(schedule_id: str):
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            sched = await db.get(AnsibleSchedule, schedule_id)
+            if sched is not None:
+                await db.delete(sched)
+    return redirect(url_for("ansible.schedules"))
+
+
+@ansible_bp.post("/schedules/<schedule_id>/run-now")
+@require_role(UserRole.operator)
+async def run_schedule_now(schedule_id: str):
+    from steward.ansible.runner import trigger_run
+
+    async with current_app.db_sessionmaker() as db:
+        sched = await db.get(AnsibleSchedule, schedule_id)
+        if sched is None:
+            return "Not found", 404
+        source_name, playbook_path = sched.source_name, sched.playbook_path
+        inventory_scope, params = sched.inventory_scope, sched.params
+
+    run, _source, err = await trigger_run(
+        current_app._get_current_object(),  # type: ignore[attr-defined]
+        source_name=source_name, playbook_path=playbook_path,
+        inventory_scope=inventory_scope, params=params,
+        triggered_by=session["user_id"],
+    )
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            fresh = await db.get(AnsibleSchedule, schedule_id)
+            if fresh:
+                fresh.last_run_at = datetime.now(timezone.utc)
+                fresh.last_run_id = run.id if run else None
+                fresh.last_error = err
+    if run:
+        return redirect(url_for("ansible.run_detail", run_id=run.id))
+    return redirect(url_for("ansible.schedules"))
+
+
+# ── In-app playbook authoring/editor (admin only) ─────────────────────────────
+
+def _editable_sources() -> list[dict]:
+    return [s for s in _get_sources() if src_module.is_editable_source(s)]
+
+
+def _editable_source(name: str) -> dict | None:
+    return next((s for s in _editable_sources() if s["name"] == name), None)
+
+
+@ansible_bp.get("/playbooks/new")
+@require_role(UserRole.admin)
+async def new_playbook():
+    sources = _editable_sources()
+    return await render_template(
+        "ansible/playbook_editor.html",
+        editable_sources=sources, editing=False,
+        source_name=(sources[0]["name"] if sources else ""),
+        playbook_path="", content=_NEW_PLAYBOOK_TEMPLATE, error=None,
+    )
+
+
+@ansible_bp.get("/playbooks/edit/<source_name>/<path:playbook_path>")
+@require_role(UserRole.admin)
+async def edit_playbook(source_name: str, playbook_path: str):
+    source = _editable_source(source_name)
+    if source is None:
+        return "Source is not editable", 404
+    content = src_module.read_playbook(source["path"], playbook_path)
+    if content is None:
+        return "Playbook not found", 404
+    return await render_template(
+        "ansible/playbook_editor.html",
+        editable_sources=_editable_sources(), editing=True,
+        source_name=source_name, playbook_path=playbook_path,
+        content=content, error=None,
+    )
+
+
+@ansible_bp.post("/playbooks/save")
+@require_role(UserRole.admin)
+async def save_playbook():
+    form = await request.form
+    source_name = (form.get("source_name", "") or "").strip()
+    playbook_path = (form.get("playbook_path", "") or "").strip()
+    content = form.get("content", "") or ""
+    editing = (form.get("editing", "") == "1")
+
+    source = _editable_source(source_name)
+    err = None
+    if source is None:
+        err = "Source is not editable"
+    elif not playbook_path:
+        err = "Filename is required"
+    else:
+        ok, verr = src_module.validate_playbook_yaml(content)
+        if not ok:
+            err = verr
+    if err is None:
+        ok, werr = src_module.write_playbook(source["path"], playbook_path, content)
+        if not ok:
+            err = werr
+
+    if err is not None:
+        return await render_template(
+            "ansible/playbook_editor.html",
+            editable_sources=_editable_sources(), editing=editing,
+            source_name=source_name, playbook_path=playbook_path,
+            content=content, error=err,
+        ), 400
+
+    return redirect(url_for(
+        "ansible.view_playbook", source_name=source_name, playbook_path=playbook_path))
+
+
+@ansible_bp.post("/playbooks/delete")
+@require_role(UserRole.admin)
+async def delete_playbook():
+    form = await request.form
+    source = _editable_source((form.get("source_name", "") or "").strip())
+    if source is None:
+        return "Source is not editable", 400
+    ok, err = src_module.delete_playbook(source["path"], (form.get("playbook_path", "") or "").strip())
+    if not ok:
+        return err, 400
+    return redirect(url_for("ansible.browse"))
+
+
+_NEW_PLAYBOOK_TEMPLATE = """\
+---
+- name: My playbook
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Ping
+      ansible.builtin.ping:
+"""

steward/ansible/runner.py

@@ -0,0 +1,92 @@
+# steward/ansible/runner.py
+"""Shared playbook-run launcher used by the manual route, alerts, and schedules.
+
+Centralises the resolve-inventory → create AnsibleRun → launch executor flow so
+manual, alert-triggered, and scheduled runs all take the exact same path.
+"""
+from __future__ import annotations
+
+import asyncio
+import uuid
+from datetime import datetime, timezone
+
+from steward.ansible import executor, sources as src_module
+from steward.ansible.inventory_gen import (
+    fetch_scope_targets, generate_inventory, inventory_to_yaml,
+)
+from steward.models.ansible import AnsibleRun, AnsibleRunStatus
+
+
+async def trigger_run(
+    app,
+    *,
+    source_name: str,
+    playbook_path: str,
+    inventory_scope: str = "steward:all",
+    params: dict | None = None,
+    triggered_by: str | None = None,
+    inventory_content: str | None = None,
+    connection: dict | None = None,
+    secret_vars: dict | None = None,
+):
+    """Resolve inventory for the scope, persist an AnsibleRun, and launch it.
+
+    triggered_by=None marks a system/automated run (alerts, schedules).
+    If inventory_content is provided, it is used verbatim and scope resolution
+    is skipped (the caller built a bespoke inventory — e.g. host_agent deploy
+    injecting per-host tokens); inventory_scope is still recorded for display.
+    connection is an optional per-run SSH override (user/password) for
+    first-contact provisioning — passed to the executor but deliberately NOT
+    stored on the AnsibleRun row (params), so the password never lands in the DB.
+    secret_vars (sensitive run-time playbook variables) are likewise passed
+    through to the executor but never persisted.
+    Returns (run, source, error): on success error is None; on failure run is
+    None and error is a short human-readable reason.
+    """
+    sources = src_module.get_sources(app.config.get("ANSIBLE", {}))
+    source = next((s for s in sources if s["name"] == source_name), None)
+    if source is None:
+        return None, None, "Source not found"
+
+    inventory_path: str | None = None
+    if inventory_content is not None:
+        pass  # caller-supplied inventory wins; scope kept only for display
+    elif inventory_scope.startswith("steward:"):
+        async with app.db_sessionmaker() as db:
+            targets = await fetch_scope_targets(db, inventory_scope)
+        inventory_content = inventory_to_yaml(generate_inventory(targets))
+    elif inventory_scope.startswith("repo:"):
+        parts = inventory_scope.split(":", 2)
+        inventory_path = parts[2] if len(parts) == 3 else ""
+    else:
+        return None, source, "Invalid inventory_scope"
+
+    run_id = str(uuid.uuid4())
+    run = AnsibleRun(
+        id=run_id,
+        playbook_path=playbook_path,
+        inventory_path=inventory_path,
+        inventory_scope=inventory_scope,
+        source_name=source_name,
+        triggered_by=triggered_by,
+        status=AnsibleRunStatus.running,
+        started_at=datetime.now(timezone.utc),
+        params=params or None,
+    )
+    async with app.db_sessionmaker() as db:
+        async with db.begin():
+            db.add(run)
+
+    task = asyncio.create_task(
+        executor.start_run(
+            app, run_id, playbook_path, inventory_path or "",
+            source["path"], params or None, inventory_content,
+            connection=connection, secret_vars=secret_vars,
+        )
+    )
+    task.add_done_callback(
+        lambda t: t.exception() and app.logger.error(
+            "Ansible run %s raised: %s", run_id, t.exception()
+        )
+    )
+    return run, source, None

steward/ansible/scheduler.py

@@ -0,0 +1,66 @@
+# steward/ansible/scheduler.py
+"""Fire due Ansible schedules. Driven by the core ScheduledTask loop (~60s)."""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+
+from sqlalchemy import select
+
+from steward.ansible.runner import trigger_run
+from steward.models.ansible_schedule import AnsibleSchedule
+
+logger = logging.getLogger(__name__)
+
+
+def is_due(schedule: AnsibleSchedule, now: datetime) -> bool:
+    """True when an enabled schedule has never run or its interval has elapsed.
+
+    Cadence is measured from the previous *fire* time (last_run_at), not run
+    completion, so a long-running playbook doesn't cause catch-up storms.
+    """
+    if not schedule.enabled:
+        return False
+    if schedule.last_run_at is None:
+        return True
+    return (now - schedule.last_run_at).total_seconds() >= schedule.interval_seconds
+
+
+async def run_due_schedules(app) -> int:
+    """Trigger every due schedule once. Returns the number launched."""
+    now = datetime.now(timezone.utc)
+    async with app.db_sessionmaker() as db:
+        schedules = (await db.execute(
+            select(AnsibleSchedule).where(AnsibleSchedule.enabled.is_(True))
+        )).scalars().all()
+
+    fired = 0
+    for s in schedules:
+        if not is_due(s, now):
+            continue
+        run = None
+        err: str | None = None
+        try:
+            run, _source, err = await trigger_run(
+                app,
+                source_name=s.source_name,
+                playbook_path=s.playbook_path,
+                inventory_scope=s.inventory_scope,
+                params=s.params,
+                triggered_by=None,
+            )
+        except Exception:
+            logger.exception("Schedule %s (%s) failed to launch", s.id, s.name)
+            err = "launch error"
+
+        async with app.db_sessionmaker() as db:
+            async with db.begin():
+                fresh = await db.get(AnsibleSchedule, s.id)
+                if fresh:
+                    fresh.last_run_at = now
+                    fresh.last_run_id = run.id if run else None
+                    fresh.last_error = err
+        if run:
+            fired += 1
+            logger.info("Schedule %s (%s) fired run %s", s.id, s.name, run.id)
+    return fired

steward/ansible/sources.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import os
+import re
 import stat
 import tempfile
 from pathlib import Path
@@ -10,6 +11,102 @@ logger = logging.getLogger(__name__)
 
 INVENTORY_NAMES = {"hosts", "inventory", "inventory.yml", "inventory.ini"}
 
+# Variable names that should be entered masked + kept out of the DB. Matched
+# case-insensitively as a substring of the variable name.
+_SECRET_VAR_RE = re.compile(
+    r"(password|passwd|secret|token|api[_-]?key|private[_-]?key|credential)", re.I
+)
+
+# A playbook self-describes via a `# description:` magic comment (Ansible rejects
+# unknown play keys, so a comment is the portable place). First match wins.
+_DESCRIPTION_RE = re.compile(r"^\s*#\s*description:\s*(.+?)\s*$", re.I | re.M)
+# Namespaced metadata: `# steward:<key>: <value>` (category, confirm, …).
+_STEWARD_META_RE = re.compile(r"^\s*#\s*steward:(\w+):\s*(.+?)\s*$", re.I | re.M)
+_TRUTHY = {"1", "true", "yes", "on"}
+
+
+def _first_play_name(content: str) -> str:
+    import yaml
+    try:
+        plays = yaml.safe_load(content)
+    except yaml.YAMLError:
+        return ""
+    if isinstance(plays, list):
+        for play in plays:
+            if isinstance(play, dict) and play.get("name"):
+                return str(play["name"]).strip()
+    return ""
+
+
+def discover_playbook_meta(content: str) -> dict:
+    """Parse Steward's playbook metadata from magic comments.
+
+    Returns {description, category, confirm}. Sources:
+      - description: ``# description:`` (primary) or ``# steward:description:``;
+        falls back to the first play's ``name:``.
+      - category: ``# steward:category: <text>`` (free-text grouping label).
+      - confirm: ``# steward:confirm: true`` → require an explicit confirmation
+        in the run form before launching (for destructive playbooks).
+    First match wins for each key.
+    """
+    steward: dict[str, str] = {}
+    for km in _STEWARD_META_RE.finditer(content):
+        steward.setdefault(km.group(1).lower(), km.group(2).strip())
+
+    m = _DESCRIPTION_RE.search(content)
+    if m:
+        description = m.group(1).strip()
+    elif steward.get("description"):
+        description = steward["description"]
+    else:
+        description = _first_play_name(content)
+
+    return {
+        "description": description,
+        "category": steward.get("category", ""),
+        "confirm": str(steward.get("confirm", "")).strip().lower() in _TRUTHY,
+    }
+
+# Name of the always-present, read-only source of first-party playbooks shipped
+# inside the app (maintenance tasks, host-agent install). Not operator-editable.
+BUILTIN_SOURCE_NAME = "steward-builtin"
+
+# Always-present, WRITABLE local source where the in-app editor saves playbooks,
+# so a homelab user with no git workflow can author automation. Lives on the
+# persistent /data volume; created lazily on first save.
+LOCAL_SOURCE_NAME = "steward-local"
+USER_PLAYBOOK_DIR = os.environ.get("STEWARD_PLAYBOOK_DIR", "/data/ansible/playbooks")
+
+
+def _builtin_source() -> dict:
+    return {
+        "name": BUILTIN_SOURCE_NAME,
+        "type": "local",
+        "path": str((Path(__file__).parent / "bundled").resolve()),
+        "url": None,
+        "branch": "main",
+        "pull_interval_seconds": 0,
+        "http_token": "",
+    }
+
+
+def _local_source() -> dict:
+    return {
+        "name": LOCAL_SOURCE_NAME,
+        "type": "local",
+        "path": USER_PLAYBOOK_DIR,
+        "url": None,
+        "branch": "main",
+        "pull_interval_seconds": 0,
+        "http_token": "",
+    }
+
+
+def is_editable_source(source: dict) -> bool:
+    """Editable in the in-app editor: local dir sources except the read-only
+    bundled one. Git sources are GitOps (clobbered on pull) so not editable."""
+    return source.get("type") == "local" and source.get("name") != BUILTIN_SOURCE_NAME
+
 
 def get_sources(ansible_cfg: dict) -> list[dict]:
     """Return resolved source list from ansible config section.
@@ -33,7 +130,8 @@ def get_sources(ansible_cfg: dict) -> list[dict]:
     """
     sources = ansible_cfg.get("sources", [])
     cache_dir = ansible_cfg.get("cache_dir", "/var/cache/steward/ansible")
-    result = []
+    # Always expose the bundled (read-only) + local (writable) first-party sources.
+    result = [_builtin_source(), _local_source()]
     for src in sources:
         src_type = src.get("type", "local")
         if src_type == "git":
@@ -102,6 +200,129 @@ def read_playbook(source_path: str, relative_path: str) -> str | None:
     return target.read_text(errors="replace")
 
 
+def _resolve_writable(source_path: str, relative_path: str) -> tuple[Path | None, str | None]:
+    """Resolve relative_path under source_path, guarding traversal + extension.
+
+    Returns (target_path, None) on success or (None, error). The root need not
+    exist yet (steward-local is created on first save)."""
+    rel = relative_path.strip().lstrip("/")
+    if not rel:
+        return None, "Filename is required"
+    if not rel.endswith((".yml", ".yaml")):
+        return None, "Playbook filename must end in .yml or .yaml"
+    root = Path(source_path).resolve()
+    target = (root / rel).resolve()
+    try:
+        target.relative_to(root)
+    except ValueError:
+        return None, "Invalid path"
+    return target, None
+
+
+def write_playbook(source_path: str, relative_path: str, content: str) -> tuple[bool, str | None]:
+    """Write a playbook into a source, creating parent dirs. Traversal-guarded."""
+    target, err = _resolve_writable(source_path, relative_path)
+    if err:
+        return False, err
+    try:
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(content, encoding="utf-8")
+    except OSError as exc:
+        return False, f"Could not write file: {exc}"
+    return True, None
+
+
+def delete_playbook(source_path: str, relative_path: str) -> tuple[bool, str | None]:
+    """Delete a playbook from a source. Traversal-guarded; no-op if absent."""
+    target, err = _resolve_writable(source_path, relative_path)
+    if err:
+        return False, err
+    try:
+        if target.exists():
+            target.unlink()
+    except OSError as exc:
+        return False, f"Could not delete file: {exc}"
+    return True, None
+
+
+def discover_playbook_description(content: str) -> str:
+    """A human-readable description of what a playbook does (see
+    discover_playbook_meta). Returns "" if none can be determined."""
+    return discover_playbook_meta(content)["description"]
+
+
+def discover_playbook_variables(content: str) -> list[dict]:
+    """Parse a playbook and list the variables an operator can set at run time.
+
+    Surfaces two sources, in this precedence (first wins on name collision):
+      - ``vars_prompt:`` — Ansible's explicit "ask me" mechanism. ``required``
+        when it has no default; ``secret`` when ``private`` (Ansible's default
+        is private=yes) or the name looks sensitive.
+      - ``vars:`` scalar entries — shown with their default as a placeholder
+        (NOT prefilled, so an untouched field falls through to inventory/play
+        defaults rather than overriding them).
+
+    Only top-level plays are inspected — role defaults and included var files are
+    not traversed (kept simple + predictable). Returns a list of dicts:
+    {name, default, secret, required, prompt}.
+    """
+    import yaml
+    try:
+        plays = yaml.safe_load(content)
+    except yaml.YAMLError:
+        return []
+    if not isinstance(plays, list):
+        return []
+
+    out: list[dict] = []
+    seen: set[str] = set()
+
+    def add(name, default, secret, required, prompt=None):
+        if not isinstance(name, str) or name in seen:
+            return
+        seen.add(name)
+        out.append({
+            "name": name,
+            "default": "" if default is None else default,
+            "secret": secret,
+            "required": required,
+            "prompt": prompt,
+        })
+
+    for play in plays:
+        if not isinstance(play, dict):
+            continue
+        for vp in play.get("vars_prompt") or []:
+            if not isinstance(vp, dict) or "name" not in vp:
+                continue
+            name = vp["name"]
+            default = vp.get("default")
+            private = bool(vp.get("private", True))  # Ansible defaults private=yes
+            add(name, default, private or bool(_SECRET_VAR_RE.search(str(name))),
+                default is None, vp.get("prompt"))
+        play_vars = play.get("vars")
+        if isinstance(play_vars, dict):
+            for name, default in play_vars.items():
+                # Only scalar defaults map cleanly to a single input field.
+                if isinstance(default, (str, int, float, bool)) or default is None:
+                    add(name, default, bool(_SECRET_VAR_RE.search(str(name))), False)
+    return out
+
+
+def validate_playbook_yaml(content: str) -> tuple[bool, str | None]:
+    """Cheap save-time validation: parses as YAML and is a non-empty play list."""
+    import yaml
+    try:
+        data = yaml.safe_load(content)
+    except yaml.YAMLError as exc:
+        return False, f"YAML error: {exc}"
+    if data is None:
+        return False, "Playbook is empty"
+    if not isinstance(data, list):
+        return False, "A playbook must be a list of plays (top-level YAML list)"
+    return True, None
+
+
 async def git_pull(source: dict) -> None:
     """Clone the git repo if absent; pull if already present.
 

steward/app.py

@@ -1,11 +1,14 @@
 # steward/app.py
 from __future__ import annotations
 import asyncio
+import logging
 from pathlib import Path
 from quart import Quart, render_template
 from .config import load_bootstrap
 from .database import init_db
 
+logger = logging.getLogger(__name__)
+
 
 def create_app(
     config_path: Path | str | None = None,
@@ -47,11 +50,27 @@ def create_app(
             plugin_dirs=[Path(d).resolve() for d in app.config["PLUGIN_DIRS"]],
         )
 
+    # ── 3b. Encryption-at-rest: bind the encryptor, then encrypt legacy secrets ─
+    if not testing:
+        from .core.crypto import init_crypto
+        from .core.settings import migrate_plaintext_secrets, scan_undecryptable_secrets
+        init_crypto(app.config["SECRET_KEY"])
+        migrate_plaintext_secrets(app.config["DATABASE_URL"])
+        # Flag any secrets sealed under a now-lost key (see the admin banner).
+        undecryptable = scan_undecryptable_secrets(app.config["DATABASE_URL"])
+        if undecryptable:
+            logger.warning(
+                "%d stored secret(s) cannot be decrypted with the current app key "
+                "(re-enter them in Settings): %s",
+                len(undecryptable), ", ".join(undecryptable),
+            )
+
     # ── 4. Load all settings from DB → populate app.config ────────────────────
     if not testing:
         from .core.settings import (
             load_settings_sync, to_smtp_cfg, to_webhook_cfg,
             to_ansible_cfg, to_plugins_cfg, to_oidc_cfg, to_ldap_cfg,
+            to_thresholds_cfg,
         )
         _settings = load_settings_sync(app.config["DATABASE_URL"])
         app.config.update(
@@ -64,8 +83,10 @@ def create_app(
             PLUGINS=to_plugins_cfg(_settings),
             OIDC=to_oidc_cfg(_settings),
             LDAP=to_ldap_cfg(_settings),
+            THRESHOLDS=to_thresholds_cfg(_settings),
         )
     else:
+        from .core.settings import to_thresholds_cfg
         app.config.update(
             SESSION_LIFETIME_HOURS=8,
             DATA_RETENTION_DAYS=90,
@@ -76,6 +97,15 @@ def create_app(
             PLUGINS={},
             OIDC={"enabled": False},
             LDAP={"enabled": False},
+            THRESHOLDS=to_thresholds_cfg({}),
+        )
+
+    # `threshold_style(value, kind)` — degraded-value coloring for templates,
+    # reading the configurable cutoffs from app.config["THRESHOLDS"]. Available
+    # in every template (core + plugin) via the jinja global.
+    from .core.settings import threshold_style_for as _ts_for
+    app.jinja_env.globals["threshold_style"] = (
+        lambda value, kind: _ts_for(value, kind, app.config.get("THRESHOLDS", {}))
     )
 
     # ── 5. Plugin migrations ───────────────────────────────────────────────────
@@ -96,8 +126,8 @@ def create_app(
     from .auth.routes import auth_bp
     from .dashboard.routes import dashboard_bp
     from .hosts.routes import hosts_bp
-    from .ping.routes import ping_bp
-    from .dns.routes import dns_bp
+    from .monitors.routes import monitors_bp
+    from .status.routes import status_bp
     from .alerts.routes import alerts_bp
     from .ansible.routes import ansible_bp
     from .ansible.inventory_routes import inventory_bp
@@ -107,14 +137,32 @@ def create_app(
     app.register_blueprint(auth_bp)
     app.register_blueprint(dashboard_bp)
     app.register_blueprint(hosts_bp)
-    app.register_blueprint(ping_bp)
-    app.register_blueprint(dns_bp)
+    app.register_blueprint(monitors_bp)
+    app.register_blueprint(status_bp)
     app.register_blueprint(alerts_bp)
     app.register_blueprint(ansible_bp)
     app.register_blueprint(inventory_bp)
     app.register_blueprint(settings_bp)
     app.register_blueprint(audit_bp)
 
+    # Register the unified Monitor status source for the Status page. Plugins
+    # may register additional sources from setup(). Idempotent.
+    from .core.status import register_status_source, monitor_status_source
+    register_status_source(monitor_status_source)
+
+    # Publish the Ansible "run a playbook" capability so plugins (e.g. host_agent
+    # auto-deploy) can drive runs without importing the runner. Ansible is core,
+    # so this is always available; consumers still gate on has_capability().
+    from .core.capabilities import register_capability
+    from .ansible.runner import trigger_run
+    from .models.users import UserRole as _UserRole
+    register_capability(
+        "ansible.run_playbook", trigger_run,
+        label="Run Ansible playbook",
+        description="Launch an Ansible playbook run (manual, alert, schedule, or plugin-driven).",
+        required_role=_UserRole.operator,
+    )
+
     # ── 8. Build task registry ─────────────────────────────────────────────────
     app._task_registry = []
 
@@ -129,8 +177,26 @@ def create_app(
     # ── 10. Template context: inject plugin_failures into every response ───────
     @app.context_processor
     def _inject_plugin_failures():
-        from .core.plugin_manager import get_plugin_failures
-        return {"plugin_failures": get_plugin_failures()}
+        from .core.plugin_manager import get_plugin_failures, get_plugin_nav
+        from .core.settings import get_undecryptable_secrets
+        # enabled_plugins lets templates gate cross-plugin embeds (e.g. the
+        # Hosts hub only fetches the docker fragment when docker is enabled),
+        # avoiding a 404 to a route whose blueprint isn't registered.
+        enabled_plugins = {
+            name for name, cfg in (app.config.get("PLUGINS") or {}).items()
+            if isinstance(cfg, dict) and cfg.get("enabled")
+        }
+        # Sidebar entries for enabled plugins that expose a UI (get_nav). Filter
+        # by enabled so a hot-disabled plugin's link can't linger before restart.
+        plugin_nav = [e for e in get_plugin_nav() if e["plugin"] in enabled_plugins]
+        return {
+            "plugin_failures": get_plugin_failures(),
+            "enabled_plugins": enabled_plugins,
+            "plugin_nav": plugin_nav,
+            # Read from an in-memory cache (no DB hit per render); kept current by
+            # the startup scan + set_setting's discard-on-write.
+            "undecryptable_secrets": get_undecryptable_secrets(),
+        }
 
     # ── 11. Share-token middleware ─────────────────────────────────────────────
     @app.before_request
@@ -190,35 +256,9 @@ def _register_core_tasks(app: Quart) -> None:
     poll_interval = app.config.get("MONITORS_POLL_INTERVAL", 60)
     cleanup_interval = 3600  # hourly
 
-    async def run_ping_monitors():
-        from sqlalchemy import select
-        from .models.hosts import Host
-        from .monitors.ping import ping_check
-        async with app.db_sessionmaker() as session:
-            async with session.begin():
-                result = await session.execute(
-                    select(Host).where(Host.ping_enabled.is_(True))
-                )
-                for host in result.scalars().all():
-                    try:
-                        await ping_check(host, session)
-                    except Exception:
-                        app.logger.exception(f"Ping check failed for host {host.name!r}")
-
-    async def run_dns_monitors():
-        from sqlalchemy import select
-        from .models.hosts import Host
-        from .monitors.dns import dns_check
-        async with app.db_sessionmaker() as session:
-            async with session.begin():
-                result = await session.execute(
-                    select(Host).where(Host.dns_enabled.is_(True))
-                )
-                for host in result.scalars().all():
-                    try:
-                        await dns_check(host, session)
-                    except Exception:
-                        app.logger.exception(f"DNS check failed for host {host.name!r}")
+    async def run_monitor_checks():
+        from .monitors.scheduler import run_due_monitors
+        await run_due_monitors(app)
 
     async def run_cleanup():
         from .core.cleanup import run_cleanup as _cleanup
@@ -228,6 +268,10 @@ def _register_core_tasks(app: Quart) -> None:
         from .core.reports import check_and_send
         await check_and_send(app)
 
+    async def run_self_monitor():
+        from .core.self_monitor import record_self_metrics
+        await record_self_metrics(app)
+
     app._task_registry.extend([
         ScheduledTask(
             name="weekly_report_check",
@@ -235,15 +279,18 @@ def _register_core_tasks(app: Quart) -> None:
             interval_seconds=3600,
             run_on_startup=False,
         ),
+        # Watch our own fd usage so a descriptor leak surfaces as a metric/alert
+        # instead of a silent Errno 24 lockup. Cheap; plugin_metrics roll up
+        # hourly so the 60s cadence is storage-bounded.
         ScheduledTask(
-            name="ping_monitor",
-            coro_factory=run_ping_monitors,
-            interval_seconds=poll_interval,
+            name="self_monitor",
+            coro_factory=run_self_monitor,
+            interval_seconds=60,
             run_on_startup=True,
         ),
         ScheduledTask(
-            name="dns_monitor",
-            coro_factory=run_dns_monitors,
+            name="monitor_check",
+            coro_factory=run_monitor_checks,
             interval_seconds=poll_interval,
             run_on_startup=True,
         ),
@@ -276,6 +323,21 @@ def _register_core_tasks(app: Quart) -> None:
             )
         )
 
+    # Fire due Ansible schedules (recurring playbook runs). Checks every minute;
+    # each schedule's own interval gates whether it actually runs.
+    async def run_ansible_schedules():
+        from .ansible.scheduler import run_due_schedules
+        await run_due_schedules(app)
+
+    app._task_registry.append(
+        ScheduledTask(
+            name="ansible_scheduled_runs",
+            coro_factory=run_ansible_schedules,
+            interval_seconds=60,
+            run_on_startup=False,
+        )
+    )
+
 
 async def _mark_interrupted_runs(app: Quart) -> None:
     from sqlalchemy import update
@@ -286,7 +348,8 @@ async def _mark_interrupted_runs(app: Quart) -> None:
         async with session.begin():
             await session.execute(
                 update(AnsibleRun)
-                .where(AnsibleRun.status == AnsibleRunStatus.running)
+                .where(AnsibleRun.status.in_(
+                    [AnsibleRunStatus.running, AnsibleRunStatus.queued]))
                 .values(
                     status=AnsibleRunStatus.interrupted,
                     finished_at=datetime.now(timezone.utc),

steward/auth/middleware.py

@@ -1,10 +1,64 @@
 from __future__ import annotations
 import functools
-from quart import session, redirect, url_for, abort, g
+from urllib.parse import quote, urlsplit
+from quart import session, redirect, url_for, abort, g, request, Response
 from steward.models.users import UserRole
 
 _ROLE_ORDER = [UserRole.viewer, UserRole.operator, UserRole.admin]
 
+# Paths we never bounce back to after login (would loop or make no sense).
+_NO_RETURN = {"/login", "/logout", "/setup"}
+
+
+def role_meets(user_role: UserRole, minimum_role: UserRole) -> bool:
+    """True when user_role is at least minimum_role in the viewer<operator<admin order."""
+    return _ROLE_ORDER.index(user_role) >= _ROLE_ORDER.index(minimum_role)
+
+
+def safe_next_url(raw: str | None) -> str | None:
+    """Return a safe same-site relative path+query from `raw`, else None.
+
+    Accepts a relative path or a full URL (e.g. HTMX's HX-Current-URL) and
+    reduces it to path[?query]. Rejects off-site/protocol-relative targets and
+    the auth pages themselves, so it can't be abused as an open redirect.
+    """
+    if not raw:
+        return None
+    parts = urlsplit(raw)
+    # Protocol-relative (//host/…) has a netloc but no scheme — reject it; a full
+    # same-site URL (HX-Current-URL) has a scheme and we keep only its path.
+    if parts.netloc and not parts.scheme:
+        return None
+    path = parts.path
+    if parts.query:
+        path += "?" + parts.query
+    if not path.startswith("/") or path.startswith("//") or path.startswith("/\\"):
+        return None
+    if path.split("?", 1)[0] in _NO_RETURN:
+        return None
+    return path
+
+
+def _login_redirect():
+    """Redirect an unauthenticated request to /login, preserving where to return.
+
+    For HTMX requests the destination is the page the user is on (HX-Current-URL)
+    and we use HX-Redirect so the whole browser navigates (not a fragment swap);
+    for normal requests it's the requested path.
+    """
+    is_htmx = request.headers.get("HX-Request") == "true"
+    nxt = safe_next_url(request.headers.get("HX-Current-URL")) if is_htmx else None
+    if nxt is None:
+        nxt = safe_next_url(request.full_path)
+    login_url = url_for("auth.login")
+    if nxt:
+        login_url = f"{login_url}?next={quote(nxt, safe='/')}"
+    if is_htmx:
+        resp = Response("", status=401)
+        resp.headers["HX-Redirect"] = login_url
+        return resp
+    return redirect(login_url)
+
 
 def require_role(minimum_role: UserRole):
     """Decorator: requires authenticated user with at least minimum_role.
@@ -18,7 +72,7 @@ def require_role(minimum_role: UserRole):
                 return await f(*args, **kwargs)
             user_id = session.get("user_id")
             if not user_id:
-                return redirect(url_for("auth.login"))
+                return _login_redirect()
             user_role = UserRole(session.get("user_role", "viewer"))
             if _ROLE_ORDER.index(user_role) < _ROLE_ORDER.index(minimum_role):
                 abort(403)

steward/auth/routes.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 import bcrypt
 from quart import Blueprint, render_template, request, redirect, url_for, session
 from sqlalchemy import select, func
-from steward.auth.middleware import login_user, logout_user
+from steward.auth.middleware import login_user, logout_user, safe_next_url
 from steward.models.users import User, UserRole
 
 auth_bp = Blueprint("auth", __name__)
@@ -54,7 +54,8 @@ async def login():
         return redirect(url_for("auth.setup"))
     oidc_cfg = current_app.config.get("OIDC", {})
     return await render_template("auth/login.html",
-                                 oidc_enabled=oidc_cfg.get("enabled", False))
+                                 oidc_enabled=oidc_cfg.get("enabled", False),
+                                 next_url=safe_next_url(request.args.get("next")))
 
 
 @auth_bp.post("/login")
@@ -65,6 +66,7 @@ async def login_post():
     username = form.get("username", "").strip()
     password_str = form.get("password", "")
     password = password_str.encode()
+    dest = safe_next_url(form.get("next")) or url_for("dashboard.index")
 
     # Try local auth first
     user = await get_user_by_username(current_app, username)
@@ -72,7 +74,7 @@ async def login_post():
         login_user(user)
         await log_audit(current_app, user.id, user.username, "auth.login",
                         detail={"method": "local", "role": user.role.value})
-        return redirect(url_for("dashboard.index"))
+        return redirect(dest)
 
     # Try LDAP fallback if enabled
     ldap_cfg = current_app.config.get("LDAP", {})
@@ -86,13 +88,14 @@ async def login_post():
             login_user(user)
             await log_audit(current_app, user.id, user.username, "auth.login",
                             detail={"method": "ldap", "role": user.role.value})
-            return redirect(url_for("dashboard.index"))
+            return redirect(dest)
 
     oidc_cfg = current_app.config.get("OIDC", {})
     return await render_template(
         "auth/login.html",
         error="Invalid credentials",
         oidc_enabled=oidc_cfg.get("enabled", False),
+        next_url=safe_next_url(form.get("next")),
     ), 400
 
 
@@ -112,6 +115,7 @@ async def login_oidc():
     nonce = generate_state()
     session["oidc_state"] = state
     session["oidc_nonce"] = nonce
+    session["oidc_next"] = safe_next_url(request.args.get("next"))
     redirect_uri = url_for("auth.login_oidc_callback", _external=True)
     url = build_authorize_url(
         doc, oidc_cfg["client_id"], redirect_uri,
@@ -170,7 +174,8 @@ async def login_oidc_callback():
     login_user(user)
     await log_audit(current_app, user.id, user.username, "auth.login",
                     detail={"method": "oidc", "role": role})
-    return redirect(url_for("dashboard.index"))
+    dest = safe_next_url(session.pop("oidc_next", None)) or url_for("dashboard.index")
+    return redirect(dest)
 
 
 @auth_bp.get("/logout")
@@ -184,7 +189,10 @@ async def setup():
     from quart import current_app
     if await get_user_count(current_app) > 0:
         return redirect(url_for("auth.login"))
-    return await render_template("auth/setup.html")
+    # Pre-fill the public URL with however the operator reached this page — the
+    # common case is correct, and it's what agent installs / share links use.
+    return await render_template(
+        "auth/setup.html", default_url=request.host_url.rstrip("/"))
 
 
 @auth_bp.post("/setup")
@@ -201,9 +209,16 @@ async def setup_post():
         return await render_template("auth/setup.html", error="All fields required"), 400
     pw_hash = bcrypt.hashpw(password, bcrypt.gensalt()).decode()
     user = User(username=username, email=email, password_hash=pw_hash, role=UserRole.admin)
+    steward_url = (form.get("public_base_url", "") or "").strip().rstrip("/")
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
             db.add(user)
+            if steward_url:
+                from steward.core.settings import set_setting
+                await set_setting(db, "general.public_base_url", steward_url)
+    if steward_url:
+        # Apply immediately so agent installs / share links use it without a restart.
+        current_app.config["PUBLIC_BASE_URL"] = steward_url
     login_user(user)
     await log_audit(current_app, user.id, user.username, "auth.setup",
                     detail={"username": username})

steward/config.py

@@ -75,13 +75,27 @@ def load_bootstrap(config_path: Path | str | None = None) -> dict[str, Any]:
 
 
 def _resolve_secret_key(raw: dict) -> str:
-    """Resolve secret_key: env var → file → auto-generate."""
+    """Resolve secret_key: env var → file → auto-generate.
+
+    Refuses to start if a new key must be generated but cannot be persisted: an
+    ephemeral key changes on every restart, which silently renders every
+    encrypted secret (managed SSH key, SMTP/OIDC/LDAP credentials) unrecoverable.
+    Failing loudly with a fix beats limping along and losing data on the next
+    boot — exactly the footgun that bit the vdnt-docker02 deployment.
+    """
     from_env = _env("SECRET_KEY") or raw.get("secret_key")
     if from_env:
         return from_env
 
     if _SECRET_KEY_FILE.exists():
+        try:
             key = _SECRET_KEY_FILE.read_text().strip()
+        except OSError as exc:
+            raise RuntimeError(
+                f"App secret key file {_SECRET_KEY_FILE} exists but cannot be read "
+                f"({exc}). Make it readable by the container user (uid 1000), or set "
+                f"STEWARD_SECRET_KEY."
+            ) from exc
         if key:
             return key
 
@@ -89,11 +103,16 @@ def _resolve_secret_key(raw: dict) -> str:
     try:
         _SECRET_KEY_FILE.parent.mkdir(parents=True, exist_ok=True)
         _SECRET_KEY_FILE.write_text(key)
-        logger.info("Generated new secret key and saved to %s", _SECRET_KEY_FILE)
     except OSError as exc:
-        logger.warning(
-            "Could not write secret key to %s (%s). "
-            "Key will not persist across restarts.",
-            _SECRET_KEY_FILE, exc,
-        )
+        raise RuntimeError(
+            f"Generated a new app secret key but could not persist it to "
+            f"{_SECRET_KEY_FILE} ({exc}). An ephemeral key changes on every restart, "
+            f"which makes all encrypted secrets (managed SSH key, SMTP/OIDC/LDAP "
+            f"credentials) unrecoverable. Fix one of:\n"
+            f"  • set STEWARD_SECRET_KEY to a stable value "
+            f"(recommended for Swarm / multi-node), or\n"
+            f"  • make {_SECRET_KEY_FILE.parent} writable by the container user "
+            f"(uid 1000)."
+        ) from exc
+    logger.info("Generated new secret key and saved to %s", _SECRET_KEY_FILE)
     return key

steward/core/capabilities.py

@@ -0,0 +1,95 @@
+# steward/core/capabilities.py
+"""Plugin/core capability registry — opportunistic, decoupled synergy.
+
+A capability is a named, permission-gated action that one part of the system
+(core module or plugin) publishes and another can discover + invoke WITHOUT a
+hard import. The publisher registers a callable under a string key; a consumer
+checks `has_capability(key)` (graceful degradation — the synergy is a bonus,
+never a requirement) and calls `invoke_capability(key, actor_role, ...)`.
+
+First consumer: the host_agent plugin invokes "ansible.run_playbook" to deploy
+its agent via Ansible instead of importing the Ansible runner directly.
+
+Security: every capability declares a required_role; invoke_capability enforces
+the caller's role meets it, so a low-privilege context can't drive a privileged
+action in another module.
+"""
+from __future__ import annotations
+
+import inspect
+import logging
+from dataclasses import dataclass
+from typing import Callable
+
+from steward.auth.middleware import role_meets
+from steward.models.users import UserRole
+
+logger = logging.getLogger(__name__)
+
+
+class CapabilityUnavailable(Exception):
+    """Raised when an unknown capability key is invoked."""
+
+
+class CapabilityForbidden(Exception):
+    """Raised when the actor's role is below the capability's required_role."""
+
+
+@dataclass
+class Capability:
+    key: str
+    fn: Callable
+    label: str
+    description: str
+    required_role: UserRole
+
+
+_CAPABILITIES: dict[str, Capability] = {}
+
+
+def register_capability(
+    key: str,
+    fn: Callable,
+    *,
+    label: str,
+    description: str = "",
+    required_role: UserRole = UserRole.admin,
+) -> None:
+    """Publish a capability. Last registration for a key wins (idempotent re-register)."""
+    _CAPABILITIES[key] = Capability(key, fn, label, description, required_role)
+
+
+def has_capability(key: str) -> bool:
+    return key in _CAPABILITIES
+
+
+def get_capability(key: str) -> Capability | None:
+    return _CAPABILITIES.get(key)
+
+
+def list_capabilities() -> list[Capability]:
+    return list(_CAPABILITIES.values())
+
+
+def clear_capabilities() -> None:
+    """Reset the registry (tests)."""
+    _CAPABILITIES.clear()
+
+
+async def invoke_capability(key: str, actor_role: UserRole, /, *args, **kwargs):
+    """Invoke a registered capability after a role check. Awaits async callables.
+
+    Raises CapabilityUnavailable if the key isn't registered, CapabilityForbidden
+    if actor_role is insufficient.
+    """
+    cap = _CAPABILITIES.get(key)
+    if cap is None:
+        raise CapabilityUnavailable(key)
+    if not role_meets(actor_role, cap.required_role):
+        raise CapabilityForbidden(
+            f"capability {key!r} requires role {cap.required_role.value}"
+        )
+    result = cap.fn(*args, **kwargs)
+    if inspect.isawaitable(result):
+        result = await result
+    return result

steward/core/cleanup.py

@@ -5,8 +5,7 @@ from typing import TYPE_CHECKING
 
 from sqlalchemy import delete
 
-from steward.models.monitors import DnsResult, PingResult
-from steward.models.metrics import PluginMetric
+from steward.models.monitors import MonitorResult
 from steward.models.ansible import AnsibleRun
 
 if TYPE_CHECKING:
@@ -16,16 +15,16 @@ logger = logging.getLogger(__name__)
 
 
 async def run_cleanup(app: "Quart") -> None:
-    """Delete rows older than DATA_RETENTION_DAYS from time-series tables."""
+    """Delete rows older than DATA_RETENTION_DAYS from time-series tables, then
+    run Docker-specific rollup + retention (delegated to the docker plugin)."""
     retention_days: int = app.config.get("DATA_RETENTION_DAYS", 90)
-    cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
+    now = datetime.now(timezone.utc)
+    cutoff = now - timedelta(days=retention_days)
 
     async with app.db_sessionmaker() as session:
         async with session.begin():
             for model, ts_col in [
-                (PingResult, PingResult.probed_at),
-                (DnsResult, DnsResult.resolved_at),
-                (PluginMetric, PluginMetric.recorded_at),
+                (MonitorResult, MonitorResult.checked_at),
                 (AnsibleRun, AnsibleRun.started_at),
             ]:
                 result = await session.execute(
@@ -33,3 +32,58 @@ async def run_cleanup(app: "Quart") -> None:
                 )
                 if result.rowcount:
                     logger.info(f"Pruned {result.rowcount} rows from {model.__tablename__}")
+
+        # plugin_metrics is NOT blanket-deleted here — it's rolled up to hourly
+        # then pruned, so multi-week host history stays cheap.
+        await _run_metrics_retention(session, now)
+        await _run_docker_retention(session, now)
+
+
+async def _run_metrics_retention(session, now: datetime) -> None:
+    """Roll up + prune plugin_metrics (raw → hourly → gone). Windows read fresh
+    from settings each run (rule 25 — UI change takes effect next cleanup, no
+    restart). get_setting's SELECT autobegins, so read inside the begin block."""
+    from steward.core.metrics_retention import rollup_plugin_metrics
+    from steward.core.settings import get_setting
+
+    async with session.begin():
+        raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7)
+        rollup_days = int(await get_setting(session, "metrics.retention.rollup_days") or 90)
+        counts = await rollup_plugin_metrics(
+            session, raw_days=raw_days, rollup_days=rollup_days, now=now,
+        )
+    if counts and any(counts.values()):
+        logger.info("Metrics retention: %s", counts)
+
+
+async def _run_docker_retention(session, now: datetime) -> None:
+    """Drive the docker plugin's rollup + prune via its capability, if loaded.
+
+    Windows are read fresh from settings each run (rule 25 — a change in the
+    Settings UI takes effect on the next hourly cleanup, no restart). Kept in its
+    own transaction so a docker-side failure can't roll back the generic prune
+    above. No-op when the docker plugin is disabled (capability absent).
+    """
+    from steward.core.capabilities import has_capability, invoke_capability
+    if not has_capability("docker.run_retention"):
+        return
+    from steward.core.settings import get_setting
+    from steward.models.users import UserRole
+
+    # Reads + rollup/prune share one transaction — get_setting's SELECT would
+    # otherwise autobegin one, making a later session.begin() raise.
+    async with session.begin():
+        raw_days = int(await get_setting(session, "docker.retention.metrics_raw_days") or 7)
+        rollup_days = int(await get_setting(session, "docker.retention.metrics_rollup_days") or 90)
+        events_days = int(await get_setting(session, "docker.retention.events_days") or 30)
+        logs_days = int(await get_setting(session, "docker.logs.retention_days") or 3)
+        logs_cap = int(
+            await get_setting(session, "docker.logs.max_bytes_per_container") or 5_000_000)
+        counts = await invoke_capability(
+            "docker.run_retention", UserRole.viewer, session,
+            events_days=events_days, metrics_raw_days=raw_days,
+            metrics_rollup_days=rollup_days, logs_retention_days=logs_days,
+            logs_max_bytes_per_container=logs_cap, now=now,
+        )
+    if counts and any(counts.values()):
+        logger.info("Docker retention: %s", counts)

steward/core/crypto.py

@@ -0,0 +1,123 @@
+# steward/core/crypto.py
+"""Encryption-at-rest for sensitive settings (smtp/oidc/ldap/ansible secrets).
+
+Values are encrypted with Fernet (AES-128-CBC + HMAC) using a key derived from
+the app secret key (the same /data/secret.key used for sessions). Encrypted
+values are stored with an ``enc:v1:`` prefix so reads can transparently tell
+ciphertext from legacy plaintext during/after migration.
+
+Key-loss caveat: if the app secret key is lost or changed, encrypted secrets
+become unrecoverable — they must be re-entered. This ties secret recovery to
+the same key the rest of the app already depends on; back it up with the data.
+"""
+from __future__ import annotations
+
+import base64
+import hashlib
+import logging
+import os
+from pathlib import Path
+
+from cryptography.fernet import Fernet, InvalidToken
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+logger = logging.getLogger(__name__)
+
+ENC_PREFIX = "enc:v1:"
+_SECRET_KEY_FILE = Path("/data/secret.key")
+_fernet: Fernet | None = None
+
+
+def _make_fernet(secret_key: str) -> Fernet:
+    # Derive a stable 32-byte Fernet key from the app secret (any length string).
+    digest = hashlib.sha256(secret_key.encode("utf-8")).digest()
+    return Fernet(base64.urlsafe_b64encode(digest))
+
+
+def init_crypto(secret_key: str) -> None:
+    """Bind the encryptor to the app's secret key (called once at startup)."""
+    global _fernet
+    if secret_key:
+        _fernet = _make_fernet(secret_key)
+
+
+def _resolve_secret_key() -> str | None:
+    """Fallback key resolution mirroring config (env → /data/secret.key)."""
+    k = os.environ.get("STEWARD_SECRET_KEY")
+    if k:
+        return k
+    try:
+        if _SECRET_KEY_FILE.exists():
+            v = _SECRET_KEY_FILE.read_text().strip()
+            if v:
+                return v
+    except OSError:
+        pass
+    return None
+
+
+def _get() -> Fernet | None:
+    global _fernet
+    if _fernet is None:
+        k = _resolve_secret_key()
+        if k:
+            _fernet = _make_fernet(k)
+    return _fernet
+
+
+def is_encrypted(value) -> bool:
+    return isinstance(value, str) and value.startswith(ENC_PREFIX)
+
+
+def encrypt_secret(plaintext: str) -> str:
+    """Return an ``enc:v1:`` token, or the input unchanged if empty / no key."""
+    if not plaintext:
+        return plaintext
+    f = _get()
+    if f is None:
+        logger.warning("No secret key available — storing a secret in PLAINTEXT")
+        return plaintext
+    return ENC_PREFIX + f.encrypt(plaintext.encode("utf-8")).decode("ascii")
+
+
+def decrypt_secret(value: str, *, context: str = "") -> str:
+    """Decrypt an ``enc:v1:`` token; pass through plaintext / undecryptable values.
+
+    context (e.g. the setting key) is only used to name the value in the
+    wrong-key log line, so an operator knows exactly which secret to re-enter.
+    """
+    if not is_encrypted(value):
+        return value
+    f = _get()
+    if f is None:
+        return value
+    try:
+        return f.decrypt(value[len(ENC_PREFIX):].encode("ascii")).decode("utf-8")
+    except InvalidToken:
+        logger.error("Could not decrypt stored secret %s (wrong/rotated key — re-enter it)",
+                     context or "(unknown setting)")
+        return value
+
+
+def generate_ssh_keypair(comment: str = "steward-managed") -> tuple[str, str]:
+    """Mint a fresh ed25519 keypair for Steward's managed Ansible identity.
+
+    Returns (private_openssh_pem, public_openssh_line). ed25519 is small, fast,
+    and universally supported by modern OpenSSH. The private key is unencrypted
+    OpenSSH PEM (Steward stores it encrypted-at-rest itself); the public key is
+    the single-line ``ssh-ed25519 AAAA… comment`` form for authorized_keys.
+    """
+    key = Ed25519PrivateKey.generate()
+    private_pem = key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.OpenSSH,
+        encryption_algorithm=serialization.NoEncryption(),
+    ).decode("ascii")
+    public_line = key.public_key().public_bytes(
+        encoding=serialization.Encoding.OpenSSH,
+        format=serialization.PublicFormat.OpenSSH,
+    ).decode("ascii")
+    if comment:
+        public_line = f"{public_line} {comment}"
+    return private_pem, public_line

steward/core/metrics_retention.py

@@ -0,0 +1,112 @@
+"""Bound plugin_metrics growth: roll old raw samples up to hourly, prune the rest.
+
+plugin_metrics grows by (sources × resources × sample cadence) — host agents push
+host-level + per-core/mount/iface sub-resources every ~30s, so a fleet accrues
+millions of rows. We keep raw samples for a short window, aggregate everything
+older into hourly averages (plugin_metrics_hourly) and delete the raw rows, then
+prune hourly beyond a longer window. Charts read raw for the recent part of a
+range and hourly for the older part.
+
+Driven by the core cleanup task (steward.core.cleanup). Runs inside the caller's
+open transaction; never opens or commits its own.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+
+def _hour_floor(dt: datetime) -> datetime:
+    """Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
+    return dt.replace(minute=0, second=0, microsecond=0)
+
+
+def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
+    """Hour-aligned boundary below which raw metrics get rolled up + deleted.
+
+    Aligning to the hour means we only roll up *whole* elapsed hours — a bucket
+    is never split across the keep/roll boundary, so a re-run can't produce a
+    partial-then-complete duplicate for the same hour.
+    """
+    return _hour_floor(now - timedelta(days=raw_days))
+
+
+async def rollup_plugin_metrics(
+    session,
+    *,
+    raw_days: int,
+    rollup_days: int,
+    now: datetime | None = None,
+) -> dict:
+    """Roll up + prune plugin_metrics. Returns a counts dict for logging.
+
+    1. Aggregate plugin_metrics older than the (hour-aligned) raw window into
+       plugin_metrics_hourly (avg/max per source/resource/metric/hour), upserting
+       so a re-run is idempotent, then delete those raw rows.
+    2. Prune rolled-up rows older than the rollup window.
+    """
+    from datetime import timezone
+
+    from sqlalchemy import delete, func, select
+    from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+    from steward.models.metrics import PluginMetric, PluginMetricHourly
+
+    if now is None:
+        now = datetime.now(timezone.utc)
+
+    rolled = rolled_rows = rollup_pruned = 0
+
+    # ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
+    raw_cutoff = _rollup_cutoff(now, raw_days)
+    hour = func.date_trunc("hour", PluginMetric.recorded_at)
+    agg = (
+        select(
+            PluginMetric.source_module,
+            PluginMetric.resource_name,
+            PluginMetric.metric_name,
+            hour.label("bucket"),
+            func.avg(PluginMetric.value).label("value_avg"),
+            func.max(PluginMetric.value).label("value_max"),
+            func.count().label("sample_count"),
+        )
+        .where(PluginMetric.recorded_at < raw_cutoff)
+        .group_by(
+            PluginMetric.source_module, PluginMetric.resource_name,
+            PluginMetric.metric_name, hour,
+        )
+    )
+    for r in (await session.execute(agg)).all():
+        avg_v = float(r.value_avg or 0.0)
+        max_v = float(r.value_max or 0.0)
+        cnt = int(r.sample_count or 0)
+        await session.execute(
+            pg_insert(PluginMetricHourly)
+            .values(
+                source_module=r.source_module, resource_name=r.resource_name,
+                metric_name=r.metric_name, bucket=r.bucket,
+                value_avg=avg_v, value_max=max_v, sample_count=cnt,
+            )
+            .on_conflict_do_update(
+                constraint="uq_plugin_metrics_hourly_bucket",
+                set_={"value_avg": avg_v, "value_max": max_v, "sample_count": cnt},
+            )
+        )
+        rolled += 1
+        rolled_rows += cnt
+    if rolled:
+        await session.execute(
+            delete(PluginMetric).where(PluginMetric.recorded_at < raw_cutoff)
+        )
+
+    # ── 2. Prune rolled-up rows beyond the rollup window ──
+    rollup_cutoff = now - timedelta(days=rollup_days)
+    res = await session.execute(
+        delete(PluginMetricHourly).where(PluginMetricHourly.bucket < rollup_cutoff)
+    )
+    rollup_pruned = res.rowcount or 0
+
+    return {
+        "buckets_rolled": rolled,
+        "raw_rows_rolled": rolled_rows,
+        "rollup_pruned": rollup_pruned,
+    }

steward/core/plugin_manager.py

@@ -24,12 +24,52 @@ _LOADED_PLUGINS: set[str] = set()
 # Track plugins that failed to load: name → human-readable reason.
 _FAILED_PLUGINS: dict[str, str] = {}
 
+# Nav entries contributed by plugins via the optional get_nav() export, so a
+# plugin's UI gets a home in the sidebar. Each entry:
+#   {"plugin": <name>, "label": str, "href": str, "section": str}
+_PLUGIN_NAV: list[dict] = []
+
 
 def get_plugin_failures() -> dict[str, str]:
     """Return a copy of the failed-plugin registry (name → error message)."""
     return dict(_FAILED_PLUGINS)
 
 
+def get_plugin_nav() -> list[dict]:
+    """Plugin-contributed sidebar entries, sorted by section then label."""
+    return sorted(_PLUGIN_NAV, key=lambda e: (e.get("section", ""), e.get("label", "")))
+
+
+def _collect_plugin_nav(name: str, module) -> None:
+    """Pull a plugin's optional get_nav() entries into the nav registry.
+
+    Tolerant by design: a missing hook is fine, and a raising or malformed hook
+    is logged and skipped — a bad nav contribution must never break loading.
+    Idempotent per plugin (drops prior entries first) so hot-reload re-runs cleanly.
+    """
+    global _PLUGIN_NAV
+    _PLUGIN_NAV = [e for e in _PLUGIN_NAV if e.get("plugin") != name]
+    if not hasattr(module, "get_nav"):
+        return
+    try:
+        items = module.get_nav() or []
+    except Exception:
+        logger.exception("Plugin %r: get_nav() raised, skipping its nav entries", name)
+        return
+    for item in items:
+        try:
+            label, href = str(item["label"]), str(item["href"])
+        except (TypeError, KeyError):
+            logger.warning("Plugin %r: malformed nav item %r, skipping", name, item)
+            continue
+        _PLUGIN_NAV.append({
+            "plugin": name,
+            "label": label,
+            "href": href,
+            "section": str(item.get("section", "Infrastructure")),
+        })
+
+
 def resolve_plugin_path(plugin_dirs: list[Path], name: str) -> Path | None:
     """Return the first plugin root that contains `name`, else None.
 
@@ -222,6 +262,7 @@ def load_plugins(app: "Quart") -> None:
 
         _FAILED_PLUGINS.pop(name, None)  # clear any stale failure from a previous reload attempt
         _LOADED_PLUGINS.add(name)
+        _collect_plugin_nav(name, module)
         logger.info("Plugin %r loaded (v%s)", name, meta.get("version", "?"))
 
 
@@ -440,6 +481,7 @@ def hot_reload_plugin(app: "Quart", name: str) -> tuple[bool, str]:
         return False, f"get_scheduled_tasks() raised: {exc}"
 
     _LOADED_PLUGINS.add(name)
+    _collect_plugin_nav(name, module)
     logger.info("Plugin %r hot-reloaded (v%s)", name, meta.get("version", "?"))
     return True, f"Plugin activated (v{meta.get('version', '?')})"
 

steward/core/reports.py

@@ -12,9 +12,8 @@ from email.message import EmailMessage
 from sqlalchemy import and_, case, func, select
 
 from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertEvent
-from steward.models.hosts import Host
 from steward.models.metrics import PluginMetric
-from steward.models.monitors import PingResult, PingStatus
+from steward.models.monitors import Monitor, MonitorResult
 
 logger = logging.getLogger(__name__)
 
@@ -28,32 +27,28 @@ async def build_report_data(app) -> dict:
     cutoff_24h = now - timedelta(hours=24)
 
     async with app.db_sessionmaker() as db:
-        # ── Uptime summary (7d) ───────────────────────────────────────────────
-        host_result = await db.execute(
-            select(Host).where(Host.ping_enabled.is_(True)).order_by(Host.name)
-        )
-        hosts = host_result.scalars().all()
+        # ── Uptime summary (7d) — per enabled monitor of any type ─────────────
+        monitors = (await db.execute(
+            select(Monitor).where(Monitor.enabled.is_(True)).order_by(Monitor.name)
+        )).scalars().all()
 
         uptime_rows = await db.execute(
             select(
-                PingResult.host_id,
-                func.count(PingResult.id).label("total"),
-                func.sum(case((PingResult.status == PingStatus.up, 1), else_=0)).label("up"),
+                MonitorResult.monitor_id,
+                func.count(MonitorResult.id).label("total"),
+                func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up"),
             )
-            .where(PingResult.probed_at >= cutoff_7d)
-            .group_by(PingResult.host_id)
+            .where(MonitorResult.checked_at >= cutoff_7d)
+            .group_by(MonitorResult.monitor_id)
         )
         uptime_map: dict[str, float | None] = {}
         for row in uptime_rows:
             pct = round(float(row.up) / float(row.total) * 100, 2) if row.total else None
-            uptime_map[row.host_id] = pct
+            uptime_map[row.monitor_id] = pct
 
-        uptime_summary = []
-        for h in hosts:
-            uptime_summary.append({
-                "name": h.name,
-                "pct_7d": uptime_map.get(h.id),
-            })
+        uptime_summary = [
+            {"name": m.name, "pct_7d": uptime_map.get(m.id)} for m in monitors
+        ]
 
         # ── Active alerts ─────────────────────────────────────────────────────
         active_result = await db.execute(
@@ -112,27 +107,27 @@ async def build_report_data(app) -> dict:
             for row in top_routers_result
         ]
 
-        # ── Hosts currently down ──────────────────────────────────────────────
-        latest_ping_subq = (
-            select(PingResult.host_id, func.max(PingResult.probed_at).label("max_at"))
-            .group_by(PingResult.host_id)
+        # ── Monitors currently down (latest result is_up = false) ─────────────
+        latest_subq = (
+            select(MonitorResult.monitor_id, func.max(MonitorResult.checked_at).label("max_at"))
+            .group_by(MonitorResult.monitor_id)
             .subquery()
         )
         down_result = await db.execute(
-            select(Host.name)
-            .join(PingResult, PingResult.host_id == Host.id)
+            select(Monitor.name)
+            .join(MonitorResult, MonitorResult.monitor_id == Monitor.id)
             .join(
-                latest_ping_subq,
+                latest_subq,
                 and_(
-                    PingResult.host_id == latest_ping_subq.c.host_id,
-                    PingResult.probed_at == latest_ping_subq.c.max_at,
+                    MonitorResult.monitor_id == latest_subq.c.monitor_id,
+                    MonitorResult.checked_at == latest_subq.c.max_at,
                 ),
             )
             .where(
-                Host.ping_enabled.is_(True),
-                PingResult.status == PingStatus.down,
+                Monitor.enabled.is_(True),
+                MonitorResult.is_up.is_(False),
             )
-            .order_by(Host.name)
+            .order_by(Monitor.name)
         )
         hosts_down = [row.name for row in down_result]
 
@@ -157,7 +152,7 @@ def _render_report_text(data: dict) -> str:
     # ── Hosts currently down ──────────────────────────────────────────────────
     down = data["hosts_down"]
     if down:
-        lines.append(f"⚠  {len(down)} host(s) currently DOWN: {', '.join(down)}")
+        lines.append(f"⚠  {len(down)} monitor(s) currently DOWN: {', '.join(down)}")
         lines.append("")
 
     # ── Uptime summary ────────────────────────────────────────────────────────
@@ -168,7 +163,7 @@ def _render_report_text(data: dict) -> str:
         pct_str = f"{pct:.2f}%" if pct is not None else "no data"
         lines.append(f"  {entry['name']:<32} {pct_str}")
     if not data["uptime_summary"]:
-        lines.append("  (no ping-enabled hosts)")
+        lines.append("  (no monitors configured)")
     lines.append("")
 
     # ── Active alerts ─────────────────────────────────────────────────────────

steward/core/scheduler.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 import asyncio
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Callable, Coroutine
 
 logger = logging.getLogger(__name__)
@@ -15,28 +15,65 @@ class ScheduledTask:
     run_on_startup: bool = False
 
 
+@dataclass
+class _DueTracker:
+    """Decides which scheduled tasks are due, with a self-overlap guard.
+
+    Pure (no asyncio, no clock of its own — `now` is passed in) so the
+    scheduling policy is unit-testable without timing races. A task whose prior
+    run is still in flight is NOT re-fired: overlapping poll runs stack up open
+    connections/subprocesses and amplify any per-poll resource use — the same
+    failure mode behind the fd-leak lockups. The skipped task is retried on the
+    next tick once it completes (its last_run isn't advanced while skipped).
+    """
+    last_run: dict[str, float] = field(default_factory=dict)
+    in_flight: set[str] = field(default_factory=set)
+
+    def due(self, tasks: list[ScheduledTask], now: float) -> list[ScheduledTask]:
+        ready: list[ScheduledTask] = []
+        for task in tasks:
+            if now - self.last_run.get(task.name, 0) < task.interval_seconds:
+                continue
+            if task.name in self.in_flight:
+                logger.warning(
+                    "Scheduled task %r still running — skipping this tick",
+                    task.name)
+                continue
+            ready.append(task)
+        return ready
+
+    def mark_started(self, task: ScheduledTask, now: float) -> None:
+        self.in_flight.add(task.name)
+        self.last_run[task.name] = now
+
+    def mark_done(self, name: str) -> None:
+        self.in_flight.discard(name)
+
+
 async def start_scheduler(tasks: list[ScheduledTask]) -> None:
     """Run scheduled tasks in a loop. Call with asyncio.create_task()."""
-    last_run: dict[str, float] = {}
+    tracker = _DueTracker()
+
+    def _spawn(task: ScheduledTask, now: float) -> None:
+        tracker.mark_started(task, now)
+        asyncio.create_task(_run_task(task, tracker))
 
     for task in tasks:
         if task.run_on_startup:
             logger.info(f"Startup task: {task.name}")
-            asyncio.create_task(_run_task(task))
-            last_run[task.name] = asyncio.get_event_loop().time()
+            _spawn(task, asyncio.get_event_loop().time())
 
     while True:
         now = asyncio.get_event_loop().time()
-        for task in tasks:
-            last = last_run.get(task.name, 0)
-            if now - last >= task.interval_seconds:
-                asyncio.create_task(_run_task(task))
-                last_run[task.name] = now
+        for task in tracker.due(tasks, now):
+            _spawn(task, now)
         await asyncio.sleep(1)
 
 
-async def _run_task(task: ScheduledTask) -> None:
+async def _run_task(task: ScheduledTask, tracker: _DueTracker) -> None:
     try:
         await task.coro_factory()
     except Exception:
         logger.exception(f"Scheduled task {task.name!r} raised an exception")
+    finally:
+        tracker.mark_done(task.name)

steward/core/self_monitor.py

@@ -0,0 +1,116 @@
+"""Self-monitoring: record Steward's own open file-descriptor usage.
+
+A leaked socket/file handle in a poll loop (see the SNMP and UniFi fd-leak
+issues) used to fail silently until the process hit its fd ceiling and
+`socket.accept()` started raising `OSError: [Errno 24] Too many open files` —
+taking the whole app down with no early warning.
+
+This turns that failure mode into an observable signal. Steward monitors other
+things; it should monitor itself. We record two metrics each tick under
+`source_module="steward"`:
+
+  • ``open_fds``      — raw count of open descriptors
+  • ``open_fds_pct``  — that count as a percentage of the soft RLIMIT_NOFILE
+
+Both flow through the normal alert pipeline, so the operator can attach an alert
+rule to either via the existing alert-rules UI. As a zero-config floor we also
+log a WARNING once usage crosses ``FD_WARN_PCT`` — a leak becomes visible even
+before any rule is set up.
+"""
+from __future__ import annotations
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+# Stdlib-only fd counting via /proc keeps this dependency-free. resource is
+# POSIX-only but always present on the Linux runtime image; guard anyway so
+# imports never explode on a dev machine.
+try:
+    import resource
+except ImportError:  # pragma: no cover - non-POSIX
+    resource = None  # type: ignore[assignment]
+
+# Warn (without needing a configured alert rule) once we're using this fraction
+# of the soft fd limit. 80% leaves headroom to act before accepts start failing.
+FD_WARN_PCT = 80.0
+
+_warned = False  # de-dupe the WARNING so a sustained leak doesn't spam the log
+
+
+def count_open_fds() -> int | None:
+    """Number of open file descriptors for this process, or None if unknown.
+
+    Reads ``/proc/self/fd`` (Linux). Returns None where /proc isn't available
+    (e.g. a macOS dev box) so callers degrade to a no-op rather than guessing.
+    """
+    try:
+        return len(os.listdir("/proc/self/fd"))
+    except OSError:
+        return None
+
+
+def fd_soft_limit() -> int | None:
+    """Soft RLIMIT_NOFILE for this process, or None if it can't be read.
+
+    None when the limit is unknown or 'unlimited' (RLIM_INFINITY) — a percentage
+    against an unbounded ceiling is meaningless, so we skip the pct metric then.
+    """
+    if resource is None:
+        return None
+    try:
+        soft, _hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    except (ValueError, OSError):
+        return None
+    if soft <= 0 or soft == resource.RLIM_INFINITY:
+        return None
+    return soft
+
+
+async def record_self_metrics(app) -> None:
+    """Record open-fd usage as Steward's own metrics; warn past the floor.
+
+    No-op (logged at debug) when fd accounting isn't available on this platform,
+    so it's safe to schedule unconditionally.
+    """
+    global _warned
+
+    fds = count_open_fds()
+    if fds is None:
+        logger.debug("self_monitor: /proc/self/fd unavailable — skipping fd metrics")
+        return
+
+    from .alerts import record_metric
+
+    soft = fd_soft_limit()
+    pct = (fds / soft * 100.0) if soft else None
+
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            await record_metric(
+                session=session,
+                source_module="steward",
+                resource_name="process",
+                metric_name="open_fds",
+                value=float(fds),
+            )
+            if pct is not None:
+                await record_metric(
+                    session=session,
+                    source_module="steward",
+                    resource_name="process",
+                    metric_name="open_fds_pct",
+                    value=pct,
+                )
+
+    if pct is not None and pct >= FD_WARN_PCT:
+        if not _warned:
+            logger.warning(
+                "Open file descriptors at %.0f%% of the soft limit (%d/%d) — "
+                "possible descriptor leak; the app will stop accepting "
+                "connections if this reaches 100%%.", pct, fds, soft)
+            _warned = True
+    else:
+        _warned = False  # recovered — re-arm the warning for the next breach
+
+    logger.debug("self_monitor: open_fds=%d soft_limit=%s pct=%s", fds, soft, pct)

steward/core/settings.py

@@ -52,13 +52,50 @@ DEFAULTS: dict[str, Any] = {
     "webhook.template": _DEFAULT_WEBHOOK_TEMPLATE,
     "ansible.sources": [],
     # Ansible credentials — global, used by every run (manual + alert-triggered).
-    # Plaintext at rest, masked in the UI (encryption-at-rest tracked separately).
+    # ssh_private_key/become/vault are encrypted at rest (see SECRET_KEYS).
     "ansible.ssh_private_key": "",
     "ansible.become_password": "",
     "ansible.vault_password": "",
+    # Public half of the managed keypair — non-secret, displayed so it can be
+    # sprayed onto hosts (provisioning installs it into ~steward/.ssh).
+    "ansible.ssh_public_key": "",
+    # Default SSH login for steady-state runs — the dedicated account
+    # provisioning creates. Acts as a floor (--user); a target's ansible_user
+    # or a per-run bootstrap override still wins.
+    "ansible.ssh_user": "steward",
     "ansible.host_key_checking": False,
+    # Max simultaneous playbook runs; extra runs queue. Applied at app start.
+    "ansible.max_concurrent_runs": 3,
     "ping.threshold.good_ms": 50,
     "ping.threshold.warn_ms": 200,
+    # Degraded/critical cutoffs for metric coloring (warn=amber, crit=red).
+    # cpu/mem/disk/load are percentages (load is load-per-core %); temp in °C.
+    # uptime is "higher is better" so its warn/crit are floors. Ping latency
+    # reuses ping.threshold.good_ms/warn_ms above.
+    "thresholds.cpu_warn": 80, "thresholds.cpu_crit": 90,
+    "thresholds.mem_warn": 80, "thresholds.mem_crit": 90,
+    "thresholds.disk_warn": 80, "thresholds.disk_crit": 90,
+    "thresholds.load_warn": 80, "thresholds.load_crit": 100,
+    "thresholds.temp_warn": 70, "thresholds.temp_crit": 85,
+    "thresholds.uptime_warn": 99.0, "thresholds.uptime_crit": 95.0,
+    # Docker time-series retention (rule 25 — tunable, no restart). Raw 30s
+    # samples are heavy, so keep a short raw window then roll up to hourly
+    # averages kept much longer; lifecycle events are light, keep a month.
+    "docker.retention.metrics_raw_days": 7,
+    "docker.retention.metrics_rollup_days": 90,
+    "docker.retention.events_days": 30,
+    # Container logs (m79): on by default for every container (operator
+    # preference). `exclude` names containers the server drops on ingest; the
+    # per-container ring bounds storage (rotate oldest past whichever of ~age or
+    # ~bytes hits first — a chatty container just keeps a shorter window).
+    "docker.logs.enabled": True,
+    "docker.logs.exclude": [],
+    "docker.logs.retention_days": 3,
+    "docker.logs.max_bytes_per_container": 5_000_000,
+    # Host/plugin metrics retention (plugin_metrics): keep a short raw window at
+    # the agent's ~30s cadence, then roll up to hourly averages kept much longer.
+    "metrics.retention.raw_days": 7,
+    "metrics.retention.rollup_days": 90,
     "plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml",
     # Default-enabled plugins. These are the generic, non-vendor-specific
     # bundled plugins (protocols/standards, not a single product) — useful on
@@ -102,6 +139,56 @@ DEFAULTS: dict[str, Any] = {
     "reports.last_sent_at": "",
 }
 
+# Settings encrypted at rest (transparent encrypt-on-write / decrypt-on-read).
+# Adding a key here makes new writes ciphertext; run migrate_plaintext_secrets
+# to convert any existing plaintext rows.
+SECRET_KEYS: set[str] = {
+    "smtp.password",
+    "oidc.client_secret",
+    "ldap.bind_password",
+    "ansible.ssh_private_key",
+    "ansible.become_password",
+    "ansible.vault_password",
+}
+
+
+def _decode(value: Any, key: str = "") -> Any:
+    """Decrypt a stored value if it's an encrypted token; else pass through.
+
+    key is passed through to the decrypt log so a wrong-key failure names the
+    exact setting that needs re-entering.
+    """
+    from steward.core.crypto import decrypt_secret, is_encrypted
+    return decrypt_secret(value, context=key) if is_encrypted(value) else value
+
+
+# Secret settings that are stored as ciphertext but won't decrypt with the
+# current app key (key rotated or lost). Surfaced as an admin banner so the
+# operator knows precisely which secrets to re-enter — instead of finding out
+# only when something that uses one fails. Refreshed at startup
+# (scan_undecryptable_secrets) and kept live: re-entering a secret clears it
+# without a restart (set_setting discards it on a fresh write).
+_undecryptable_secrets: set[str] = set()
+
+
+def get_undecryptable_secrets() -> list[str]:
+    """Sorted keys whose stored ciphertext won't decrypt (for the UI banner)."""
+    return sorted(_undecryptable_secrets)
+
+
+def _is_undecryptable(stored: Any, key: str = "") -> bool:
+    """True iff `stored` is an encrypted token that fails to decrypt.
+
+    A failed decrypt returns the ciphertext unchanged (still enc-prefixed), so a
+    value that is still encrypted after a decrypt attempt is undecryptable.
+    """
+    from steward.core.crypto import decrypt_secret, is_encrypted
+    return (
+        isinstance(stored, str)
+        and is_encrypted(stored)
+        and is_encrypted(decrypt_secret(stored, context=key))
+    )
+
 
 # ─────────────────────────────────────────────────────────────────────────────
 # Async helpers (use inside request handlers / scheduled tasks)
@@ -115,27 +202,37 @@ async def get_setting(session: AsyncSession, key: str) -> Any:
     row = result.scalar_one_or_none()
     if row is None:
         return DEFAULTS.get(key)
-    return json.loads(row.value_json)
+    return _decode(json.loads(row.value_json), key)
 
 
 async def set_setting(session: AsyncSession, key: str, value: Any) -> None:
-    """Upsert a setting. Call inside an active transaction."""
+    """Upsert a setting (encrypting secret keys at rest). Call in a transaction."""
+    to_store = value
+    if key in SECRET_KEYS and isinstance(value, str) and value:
+        from steward.core.crypto import encrypt_secret
+        to_store = encrypt_secret(value)
     result = await session.execute(
         select(AppSetting).where(AppSetting.key == key)
     )
     row = result.scalar_one_or_none()
     now = datetime.now(timezone.utc)
     if row is None:
-        session.add(AppSetting(key=key, value_json=json.dumps(value), updated_at=now))
+        session.add(AppSetting(key=key, value_json=json.dumps(to_store), updated_at=now))
     else:
-        row.value_json = json.dumps(value)
+        row.value_json = json.dumps(to_store)
         row.updated_at = now
 
+    # A fresh write of a secret is encrypted with the current key (or cleared to
+    # plaintext), so it's decryptable now — drop any stale "undecryptable" flag
+    # so the banner clears without a restart.
+    if key in SECRET_KEYS:
+        _undecryptable_secrets.discard(key)
+
 
 async def get_all_settings(session: AsyncSession) -> dict[str, Any]:
     """Return flat key→value dict with defaults filled in for missing keys."""
     result = await session.execute(select(AppSetting))
-    stored = {row.key: json.loads(row.value_json) for row in result.scalars()}
+    stored = {row.key: _decode(json.loads(row.value_json), row.key) for row in result.scalars()}
     out: dict[str, Any] = {}
     for key, default in DEFAULTS.items():
         out[key] = stored.get(key, default)
@@ -174,7 +271,10 @@ def to_ansible_cfg(settings: dict[str, Any]) -> dict:
         "ssh_private_key": settings.get("ansible.ssh_private_key", ""),
         "become_password": settings.get("ansible.become_password", ""),
         "vault_password": settings.get("ansible.vault_password", ""),
+        "ssh_public_key": settings.get("ansible.ssh_public_key", ""),
+        "ssh_user": settings.get("ansible.ssh_user", "steward"),
         "host_key_checking": settings.get("ansible.host_key_checking", False),
+        "max_concurrent_runs": settings.get("ansible.max_concurrent_runs", 3),
     }
 
 
@@ -186,6 +286,53 @@ def to_ldap_cfg(settings: dict[str, Any]) -> dict:
     return {k[len("ldap."):]: settings.get(k, DEFAULTS[k]) for k in DEFAULTS if k.startswith("ldap.")}
 
 
+def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
+    """Per-metric (warn, crit, direction) policy for degraded-value coloring.
+
+    Centralizes "what counts as degraded" so templates just name a metric kind
+    via the threshold_style() jinja global. `dir` is "high" (higher is worse,
+    e.g. CPU) or "low" (lower is worse, e.g. uptime %). Latency reuses the
+    existing ping good/warn thresholds.
+    """
+    g = settings.get
+    return {
+        "cpu":     {"warn": g("thresholds.cpu_warn", 80),  "crit": g("thresholds.cpu_crit", 90),  "dir": "high"},
+        "mem":     {"warn": g("thresholds.mem_warn", 80),  "crit": g("thresholds.mem_crit", 90),  "dir": "high"},
+        "disk":    {"warn": g("thresholds.disk_warn", 80), "crit": g("thresholds.disk_crit", 90), "dir": "high"},
+        "load":    {"warn": g("thresholds.load_warn", 80), "crit": g("thresholds.load_crit", 100), "dir": "high"},
+        "temp":    {"warn": g("thresholds.temp_warn", 70), "crit": g("thresholds.temp_crit", 85), "dir": "high"},
+        "latency": {"warn": g("ping.threshold.good_ms", 50), "crit": g("ping.threshold.warn_ms", 200), "dir": "high"},
+        "uptime":  {"warn": g("thresholds.uptime_warn", 99.0), "crit": g("thresholds.uptime_crit", 95.0), "dir": "low"},
+    }
+
+
+def threshold_style_for(value: Any, kind: str, thresholds: dict) -> str:
+    """Return an inline-style fragment (amber at warn, red at crit) for a metric.
+
+    The Python twin of the old _macros.metric_style, but reading configurable
+    cutoffs from `thresholds` (see to_thresholds_cfg) and handling both
+    directions. Empty string when normal/unknown/None — drop straight into a
+    span's style="".
+    """
+    if value is None:
+        return ""
+    th = thresholds.get(kind)
+    if not th:
+        return ""
+    warn, crit, direction = th["warn"], th["crit"], th.get("dir", "high")
+    if direction == "low":
+        if value < crit:
+            return "color:var(--red);font-weight:700;"
+        if value < warn:
+            return "color:var(--yellow);font-weight:600;"
+    else:
+        if value >= crit:
+            return "color:var(--red);font-weight:700;"
+        if value >= warn:
+            return "color:var(--yellow);font-weight:600;"
+    return ""
+
+
 def to_plugins_cfg(settings: dict[str, Any]) -> dict:
     """Assemble {plugin_name: {...config}} from all plugin.* keys."""
     result = {}
@@ -213,7 +360,7 @@ def load_settings_sync(db_url: str) -> dict[str, Any]:
         try:
             async with factory() as session:
                 result = await session.execute(select(AppSetting))
-                return {row.key: json.loads(row.value_json) for row in result.scalars()}
+                return {row.key: _decode(json.loads(row.value_json), row.key) for row in result.scalars()}
         finally:
             await engine.dispose()
 
@@ -227,6 +374,68 @@ def load_settings_sync(db_url: str) -> dict[str, Any]:
     return out
 
 
+def migrate_plaintext_secrets(db_url: str) -> int:
+    """Encrypt any existing plaintext secret rows in place. Idempotent.
+
+    Returns the number of values converted. Run once at startup after the
+    encryptor is initialised (already-encrypted rows are skipped).
+    """
+    from steward.core.crypto import encrypt_secret, is_encrypted
+
+    async def _run() -> int:
+        from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
+        engine = create_async_engine(db_url, echo=False)
+        factory = async_sessionmaker(engine, expire_on_commit=False)
+        converted = 0
+        try:
+            async with factory() as session:
+                async with session.begin():
+                    result = await session.execute(
+                        select(AppSetting).where(AppSetting.key.in_(SECRET_KEYS))
+                    )
+                    for row in result.scalars():
+                        val = json.loads(row.value_json)
+                        if isinstance(val, str) and val and not is_encrypted(val):
+                            row.value_json = json.dumps(encrypt_secret(val))
+                            converted += 1
+        finally:
+            await engine.dispose()
+        return converted
+
+    return asyncio.run(_run())
+
+
+def scan_undecryptable_secrets(db_url: str) -> list[str]:
+    """Populate the undecryptable-secrets cache from the DB; return the keys found.
+
+    Run once at startup, AFTER migrate_plaintext_secrets and init_crypto: any row
+    that's encrypted but won't decrypt with the current key was sealed under a
+    different (lost/rotated) key and must be re-entered. Surfaced via the admin
+    banner (get_undecryptable_secrets).
+    """
+    async def _run() -> list[str]:
+        from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
+        engine = create_async_engine(db_url, echo=False)
+        factory = async_sessionmaker(engine, expire_on_commit=False)
+        bad: list[str] = []
+        try:
+            async with factory() as session:
+                result = await session.execute(
+                    select(AppSetting).where(AppSetting.key.in_(SECRET_KEYS))
+                )
+                for row in result.scalars():
+                    if _is_undecryptable(json.loads(row.value_json), row.key):
+                        bad.append(row.key)
+        finally:
+            await engine.dispose()
+        return bad
+
+    global _undecryptable_secrets
+    found = asyncio.run(_run())
+    _undecryptable_secrets = set(found)
+    return sorted(found)
+
+
 # ─────────────────────────────────────────────────────────────────────────────
 # External URL helper
 # ─────────────────────────────────────────────────────────────────────────────

steward/core/status.py

@@ -0,0 +1,207 @@
+# steward/core/status.py
+"""Unified monitor-status aggregation.
+
+A single readable "is everything up?" surface (the Status page + dashboard
+widget) is assembled from heterogeneous monitors. The core unified Monitor
+entity (ping/dns/http) contributes via `monitor_status_source`; plugins may
+register their own sources from setup(). Each source is an async callable(db)
+returning normalised StatusEntry objects so the Status page never imports
+plugin tables directly.
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from typing import Awaitable, Callable
+
+from sqlalchemy import and_, case, func, select
+
+logger = logging.getLogger(__name__)
+
+HEARTBEAT_COUNT = 30  # number of recent checks shown in the heartbeat bar
+
+
+@dataclass
+class StatusEntry:
+    """One monitored thing, normalised across monitor types for display."""
+    kind: str                       # "icmp" | "tcp" | "dns" | "http" | ...
+    key: str                        # unique within kind (monitor id)
+    name: str
+    target: str = ""                # address / URL shown as subtitle
+    status: str = "pending"         # "up" | "down" | "pending"
+    last_checked: datetime | None = None
+    uptime: dict[str, float | None] = field(default_factory=dict)  # 24h/7d/30d
+    heartbeat: list[dict] = field(default_factory=list)  # oldest-first {state,title}
+    latency_ms: float | None = None
+    spark: list[float] = field(default_factory=list)     # response series
+    spark_svg: str = ""             # filled by the route via sparkline_svg()
+    tls_days: float | None = None   # days until TLS expiry (http only)
+    detail_url: str | None = None
+
+
+StatusSource = Callable[[object], Awaitable[list[StatusEntry]]]
+_SOURCES: list[StatusSource] = []
+
+
+def register_status_source(fn: StatusSource) -> None:
+    """Register a status source. Idempotent on the same callable."""
+    if fn not in _SOURCES:
+        _SOURCES.append(fn)
+
+
+def clear_status_sources() -> None:
+    """Reset the registry (used by tests)."""
+    _SOURCES.clear()
+
+
+async def collect_status(db) -> list[StatusEntry]:
+    """Gather entries from every registered source, down-first then by name.
+
+    A failing source is logged and skipped so one broken plugin can't blank
+    the whole Status page.
+    """
+    entries: list[StatusEntry] = []
+    for src in list(_SOURCES):
+        try:
+            entries.extend(await src(db))
+        except Exception:
+            logger.exception("status source %r failed", getattr(src, "__name__", src))
+    # Down first (most urgent), then pending, then up; alphabetical within.
+    order = {"down": 0, "pending": 1, "up": 2}
+    entries.sort(key=lambda e: (order.get(e.status, 3), e.kind, e.name.lower()))
+    return entries
+
+
+def sparkline_svg(values: list[float], width: int = 80, height: int = 20,
+                  stroke: str = "#6060c0") -> str:
+    """Tiny inline SVG polyline for a response-time series."""
+    vals = [v for v in values if v is not None]
+    if len(vals) < 2:
+        return f'<svg width="{width}" height="{height}"></svg>'
+    mn, mx = min(vals), max(vals)
+    if mx == mn:
+        mx = mn + 1.0
+    step = width / (len(vals) - 1)
+    pts = []
+    for i, v in enumerate(vals):
+        x = i * step
+        y = height - (v - mn) / (mx - mn) * (height - 2) - 1
+        pts.append(f"{x:.1f},{y:.1f}")
+    poly = " ".join(pts)
+    return (
+        f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
+        f'style="vertical-align:middle;">'
+        f'<polyline points="{poly}" fill="none" stroke="{stroke}" stroke-width="1.5"/>'
+        f'</svg>'
+    )
+
+
+# ── Unified monitor status source ─────────────────────────────────────────────
+
+async def _last_n_results(db, monitor_ids: list[str],
+                          n: int = HEARTBEAT_COUNT) -> dict[str, list]:
+    """Return {monitor_id: [MonitorResult]} of the last n results, oldest-first."""
+    from steward.models.monitors import MonitorResult
+    if not monitor_ids:
+        return {}
+    rn = func.row_number().over(
+        partition_by=MonitorResult.monitor_id, order_by=MonitorResult.checked_at.desc()
+    ).label("rn")
+    subq = select(MonitorResult.id, rn).where(
+        MonitorResult.monitor_id.in_(monitor_ids)
+    ).subquery()
+    res = await db.execute(
+        select(MonitorResult)
+        .join(subq, MonitorResult.id == subq.c.id)
+        .where(subq.c.rn <= n)
+        .order_by(MonitorResult.monitor_id, MonitorResult.checked_at.asc())
+    )
+    out: dict[str, list] = {mid: [] for mid in monitor_ids}
+    for row in res.scalars():
+        out[row.monitor_id].append(row)
+    return out
+
+
+async def _uptime_by_monitor(db, monitor_ids: list[str]) -> dict[str, dict]:
+    """Per-monitor uptime % over 24h/7d/30d in one grouped query."""
+    from steward.models.monitors import MonitorResult
+    if not monitor_ids:
+        return {}
+    now = datetime.now(timezone.utc)
+    c30, c7, c24 = now - timedelta(days=30), now - timedelta(days=7), now - timedelta(hours=24)
+    res = await db.execute(
+        select(
+            MonitorResult.monitor_id,
+            func.count().label("total_30d"),
+            func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up_30d"),
+            func.sum(case((MonitorResult.checked_at >= c7, 1), else_=0)).label("total_7d"),
+            func.sum(case((and_(MonitorResult.checked_at >= c7, MonitorResult.is_up.is_(True)), 1), else_=0)).label("up_7d"),
+            func.sum(case((MonitorResult.checked_at >= c24, 1), else_=0)).label("total_24h"),
+            func.sum(case((and_(MonitorResult.checked_at >= c24, MonitorResult.is_up.is_(True)), 1), else_=0)).label("up_24h"),
+        )
+        .where(MonitorResult.monitor_id.in_(monitor_ids))
+        .where(MonitorResult.checked_at >= c30)
+        .group_by(MonitorResult.monitor_id)
+    )
+
+    def _pct(up, total):
+        return round(float(up) / float(total) * 100, 2) if total else None
+
+    out: dict[str, dict] = {}
+    for r in res:
+        out[r.monitor_id] = {
+            "24h": _pct(r.up_24h, r.total_24h),
+            "7d": _pct(r.up_7d, r.total_7d),
+            "30d": _pct(r.up_30d, r.total_30d),
+        }
+    return out
+
+
+def _heartbeat_title(mtype: str, r) -> str:
+    """Type-aware tooltip for one heartbeat cell."""
+    ts = f"{r.checked_at:%H:%M:%S} UTC"
+    if not r.is_up:
+        return f"{(r.error_msg or 'Down')} — {ts}"
+    if mtype == "http":
+        code = r.status_code or "—"
+        ms = f"{r.response_ms:.0f} ms" if r.response_ms is not None else ""
+        return f"{code} · {ms} — {ts}".replace(" · — ", " — ")
+    if mtype == "dns":
+        return f"{r.resolved_ip or 'resolved'} — {ts}"
+    if r.response_ms is not None:
+        return f"{r.response_ms:.0f} ms — {ts}"
+    return f"Up — {ts}"
+
+
+async def monitor_status_source(db) -> list[StatusEntry]:
+    """Contribute every enabled Monitor (all types) to the Status page."""
+    from steward.models.monitors import Monitor
+
+    monitors = list((await db.execute(
+        select(Monitor).where(Monitor.enabled.is_(True)).order_by(Monitor.name)
+    )).scalars())
+    ids = [m.id for m in monitors]
+    recent = await _last_n_results(db, ids)
+    uptime = await _uptime_by_monitor(db, ids)
+    now = datetime.now(timezone.utc)
+
+    entries: list[StatusEntry] = []
+    for m in monitors:
+        rows = recent.get(m.id, [])
+        latest = rows[-1] if rows else None
+        status = "pending" if latest is None else ("up" if latest.is_up else "down")
+        tls_days = None
+        if latest and latest.tls_expires_at:
+            tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0
+        entries.append(StatusEntry(
+            kind=m.type, key=m.id, name=m.name, target=m.target, status=status,
+            last_checked=latest.checked_at if latest else None,
+            uptime=uptime.get(m.id, {}),
+            heartbeat=[{"state": "up" if r.is_up else "down",
+                        "title": _heartbeat_title(m.type, r)} for r in rows],
+            latency_ms=latest.response_ms if latest else None,
+            spark=[r.response_ms for r in rows if r.response_ms is not None],
+            tls_days=tls_days, detail_url="/monitors/",
+        ))
+    return entries

steward/core/widgets.py

@@ -12,12 +12,67 @@
 from __future__ import annotations
 
 WIDGET_REGISTRY: dict[str, dict] = {
-    "ping": {
-        "key": "ping",
-        "label": "Ping",
-        "description": "Live ping status and latency history for all monitored hosts",
-        "hx_url": "/ping/rows",
-        "detail_url": "/ping/",
+    "monitors": {
+        "key": "monitors",
+        "label": "Monitors",
+        "description": "Uptime checks of every type — ping, DNS, and HTTP — with up/down status and latency",
+        "hx_url": "/monitors/widget",
+        "detail_url": "/monitors/",
+        "plugin": None,
+        "poll": True,
+        "params": [
+            {
+                "key": "type_filter",
+                "label": "Type",
+                "type": "select",
+                "default": "all",
+                "options": [
+                    {"value": "all",  "label": "All types"},
+                    {"value": "icmp", "label": "Ping (ICMP)"},
+                    {"value": "tcp",  "label": "Ping (TCP)"},
+                    {"value": "dns",  "label": "DNS"},
+                    {"value": "http", "label": "HTTP"},
+                ],
+            },
+            {
+                "key": "show_down_only",
+                "label": "Display filter",
+                "type": "select",
+                "default": "no",
+                "options": [
+                    {"value": "no",  "label": "All monitors"},
+                    {"value": "yes", "label": "Failing only"},
+                ],
+            },
+            {
+                "key": "limit",
+                "label": "Max shown",
+                "type": "select",
+                "default": 10,
+                "options": [
+                    {"value": 5,  "label": "5 monitors"},
+                    {"value": 10, "label": "10 monitors"},
+                    {"value": 20, "label": "20 monitors"},
+                ],
+            },
+        ],
+    },
+    "status_overview": {
+        "key": "status_overview",
+        "label": "Overview",
+        "description": "At-a-glance summary — host count, monitor up/down/pending, and alerts",
+        "hx_url": "/status/widget",
+        "detail_url": "/status/",
+        "plugin": None,
+        "poll": True,
+        "params": [],
+    },
+    "hosts_overview": {
+        "key": "hosts_overview",
+        "label": "Hosts — Overview",
+        "description": "One row per host: monitor status + uptime and agent CPU/memory/disk, linking to the host hub",
+        "hx_url": "/hosts/overview/widget",
+        "detail_url": "/hosts/",
         "plugin": None,
         "poll": True,
         "params": [],
@@ -32,16 +87,6 @@ WIDGET_REGISTRY: dict[str, dict] = {
         "poll": False,
         "params": [],
     },
-    "dns": {
-        "key": "dns",
-        "label": "DNS",
-        "description": "DNS resolution status for all monitored hosts",
-        "hx_url": "/dns/rows",
-        "detail_url": "/dns/",
-        "plugin": None,
-        "poll": True,
-        "params": [],
-    },
     "traefik_routers": {
         "key": "traefik_routers",
         "label": "Traefik — Routers",
@@ -183,38 +228,6 @@ WIDGET_REGISTRY: dict[str, dict] = {
             },
         ],
     },
-    "http_monitors": {
-        "key": "http_monitors",
-        "label": "HTTP Monitors",
-        "description": "Synthetic endpoint checks — up/down status and response times",
-        "hx_url": "/plugins/http/widget",
-        "detail_url": "/plugins/http/",
-        "plugin": "http",
-        "poll": True,
-        "params": [
-            {
-                "key": "show_down_only",
-                "label": "Display filter",
-                "type": "select",
-                "default": "no",
-                "options": [
-                    {"value": "no",  "label": "All monitors"},
-                    {"value": "yes", "label": "Failing only"},
-                ],
-            },
-            {
-                "key": "limit",
-                "label": "Max shown",
-                "type": "select",
-                "default": 10,
-                "options": [
-                    {"value": 5,  "label": "5 monitors"},
-                    {"value": 10, "label": "10 monitors"},
-                    {"value": 20, "label": "20 monitors"},
-                ],
-            },
-        ],
-    },
     "docker_resources": {
         "key": "docker_resources",
         "label": "Docker — Resources",
@@ -250,22 +263,29 @@ WIDGET_REGISTRY: dict[str, dict] = {
     "host_resources": {
         "key": "host_resources",
         "label": "Host Agent — Resources",
-        "description": "Fleet view: CPU, memory, disk, and load per monitored host",
+        "description": "Fleet view: CPU, memory, disk, and load per host — each with a trend sparkline",
         "hx_url": "/plugins/host_agent/widget",
-        "detail_url": "/plugins/host_agent/settings/",
+        "detail_url": "/hosts/",
         "plugin": "host_agent",
         "poll": True,
         "params": [],
     },
     "host_resource_history": {
         "key": "host_resource_history",
-        "label": "Host Agent — History",
-        "description": "CPU/memory/disk history for one host",
+        "label": "Host Agent — History Graph",
+        "description": "CPU / memory / disk time-series graph for one host (the host-view chart, on your dashboard)",
         "hx_url": "/plugins/host_agent/widget/history",
-        "detail_url": "/plugins/host_agent/settings/",
+        "detail_url": "/hosts/",
         "plugin": "host_agent",
         "poll": True,
         "params": [
+            # type "host" → the edit form renders a live dropdown of hosts.
+            {
+                "key": "host_id",
+                "label": "Host",
+                "type": "host",
+                "default": "",
+            },
             {
                 "key": "hours",
                 "label": "Time range",
@@ -282,8 +302,34 @@ WIDGET_REGISTRY: dict[str, dict] = {
 }
 
 
+# Group each widget for the picker (Core monitors / Monitoring capabilities /
+# Integrations), mirroring the Settings → Plugins taxonomy. Capability plugins
+# are host/monitoring facets; traefik/unifi are discrete vendor integrations.
+_CORE_WIDGETS = {"monitors", "status_overview", "uptime_summary", "hosts_overview"}
+_INTEGRATION_PLUGINS = {"traefik", "unifi"}
+GROUP_LABELS = {
+    "core": "Core monitors",
+    "capability": "Monitoring capabilities",
+    "integration": "Integrations",
+}
+GROUP_ORDER = ["core", "capability", "integration"]
+
+
+def _group_for(w: dict) -> str:
+    if w["key"] in _CORE_WIDGETS:
+        return "core"
+    if w.get("plugin") in _INTEGRATION_PLUGINS:
+        return "integration"
+    return "capability"
+
+
+for _w in WIDGET_REGISTRY.values():
+    _w["group"] = _group_for(_w)
+
+
 def register_widget(definition: dict) -> None:
     """Register a widget at runtime (used by external plugins)."""
+    definition.setdefault("group", _group_for(definition))
     WIDGET_REGISTRY[definition["key"]] = definition
 
 

steward/dashboard/routes.py

@@ -9,7 +9,6 @@ from sqlalchemy import select, func, update
 from steward.auth.middleware import require_role, current_user_id
 from steward.models.users import UserRole
 from steward.models.hosts import Host
-from steward.models.monitors import PingResult, DnsResult
 from steward.models.dashboard import Dashboard, DashboardWidget, DashboardShareToken
 from steward.core.settings import public_base_url
 from steward.core.widgets import get_available_widgets, WIDGET_REGISTRY
@@ -38,19 +37,16 @@ def _can_edit(dash: Dashboard, user_id: str, user_role: str) -> bool:
 # ── DB helpers ────────────────────────────────────────────────────────────────
 
 async def _get_widgets(db, dashboard_id: int) -> list[DashboardWidget]:
+    # Reading order = grid order (top-to-bottom, then left-to-right). This drives
+    # DOM order, which is what the responsive single-column fallback collapses to.
     result = await db.execute(
         select(DashboardWidget)
         .where(DashboardWidget.dashboard_id == dashboard_id)
-        .order_by(DashboardWidget.position)
+        .order_by(DashboardWidget.grid_y, DashboardWidget.grid_x, DashboardWidget.id)
     )
     return list(result.scalars())
 
 
-async def _normalise_positions(db, dashboard_id: int) -> None:
-    for i, w in enumerate(await _get_widgets(db, dashboard_id)):
-        w.position = i
-
-
 async def _accessible_dashboards(db, user_id: str, user_role: str) -> list[Dashboard]:
     """All dashboards visible to this user, ordered by id."""
     result = await db.execute(select(Dashboard).order_by(Dashboard.id))
@@ -97,49 +93,15 @@ async def _get_or_create_system_default(db) -> Dashboard:
         db.add(dash)
         await db.flush()
         plugins_cfg = current_app.config.get("PLUGINS", {})
+        # Seed a 3-up grid (each widget 4 cols wide x 4 cells tall on the 12-col grid).
         for pos, w in enumerate(get_available_widgets(plugins_cfg)):
             db.add(DashboardWidget(
-                dashboard_id=dash.id, widget_key=w["key"], position=pos,
+                dashboard_id=dash.id, widget_key=w["key"],
+                grid_x=(pos % 3) * 4, grid_y=(pos // 3) * 4, grid_w=4, grid_h=4,
             ))
     return dash
 
 
-async def _get_summary_stats(db) -> dict:
-    latest_ping_subq = (
-        select(PingResult.host_id, func.max(PingResult.probed_at).label("max_at"))
-        .group_by(PingResult.host_id).subquery()
-    )
-    pr = await db.execute(
-        select(PingResult).join(
-            latest_ping_subq,
-            (PingResult.host_id == latest_ping_subq.c.host_id)
-            & (PingResult.probed_at == latest_ping_subq.c.max_at),
-        )
-    )
-    pings     = list(pr.scalars())
-    ping_up   = sum(1 for p in pings if p.status.value == "up")
-    ping_down = sum(1 for p in pings if p.status.value == "down")
-
-    latest_dns_subq = (
-        select(DnsResult.host_id, func.max(DnsResult.resolved_at).label("max_at"))
-        .group_by(DnsResult.host_id).subquery()
-    )
-    dr = await db.execute(
-        select(DnsResult).join(
-            latest_dns_subq,
-            (DnsResult.host_id == latest_dns_subq.c.host_id)
-            & (DnsResult.resolved_at == latest_dns_subq.c.max_at),
-        )
-    )
-    dns      = list(dr.scalars())
-    dns_ok   = sum(1 for d in dns if d.status.value == "resolved")
-    dns_fail = sum(1 for d in dns if d.status.value != "resolved")
-    total    = (await db.execute(select(func.count()).select_from(Host))).scalar()
-
-    return dict(ping_up=ping_up, ping_down=ping_down,
-                dns_ok=dns_ok, dns_fail=dns_fail, total_hosts=total)
-
-
 def _resolve_widgets(widgets: list[DashboardWidget]) -> list[dict]:
     plugins_cfg = current_app.config.get("PLUGINS", {})
     out = []
@@ -170,31 +132,6 @@ def _resolve_widgets(widgets: list[DashboardWidget]) -> list[dict]:
 
 # ── Edit panel helpers ────────────────────────────────────────────────────────
 
-async def _get_panels_data(db, dashboard_id: int) -> tuple:
-    result = await db.execute(select(Dashboard).where(Dashboard.id == dashboard_id))
-    dash   = result.scalar_one_or_none()
-    if dash is None:
-        return None, [], []
-    widgets     = await _get_widgets(db, dashboard_id)
-    plugins_cfg = current_app.config.get("PLUGINS", {})
-    available   = get_available_widgets(plugins_cfg)
-    # Multiple instances of the same widget type are allowed — show all available
-    resolved    = [
-        {"db": w, "defn": WIDGET_REGISTRY[w.widget_key], "title": w.title or WIDGET_REGISTRY[w.widget_key]["label"]}
-        for w in widgets if w.widget_key in WIDGET_REGISTRY
-    ]
-    return dash, resolved, available
-
-
-async def _panels_response(dashboard_id: int):
-    async with current_app.db_sessionmaker() as db:
-        dash, resolved, addable = await _get_panels_data(db, dashboard_id)
-    return await render_template(
-        "dashboard/_edit_panels.html",
-        dashboard=dash, widgets=resolved, addable=addable,
-    )
-
-
 # ── Root redirect ─────────────────────────────────────────────────────────────
 
 @dashboard_bp.get("/")
@@ -225,13 +162,20 @@ async def view(dash_id: int):
         if dash is None or not _can_view(dash, user_id, user_role):
             abort(404)
 
-        stats      = await _get_summary_stats(db)
         widgets    = await _get_widgets(db, dash_id)
         accessible = await _accessible_dashboards(db, user_id, user_role)
+        can_edit   = _can_edit(dash, user_id, user_role)
+        # Editors get the bottom-drawer picker (available widgets + host list);
+        # viewers/shared-readers don't, so skip the extra queries for them.
+        if can_edit:
+            plugins_cfg = current_app.config.get("PLUGINS", {})
+            addable = get_available_widgets(plugins_cfg)
+            hosts   = list((await db.execute(select(Host).order_by(Host.name))).scalars())
+        else:
+            addable, hosts = [], []
 
     resolved      = _resolve_widgets(widgets)
     poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
-    can_edit      = _can_edit(dash, user_id, user_role)
 
     return await render_template(
         "dashboard/index.html",
@@ -240,7 +184,8 @@ async def view(dash_id: int):
         widgets=resolved,
         poll_interval=poll_interval,
         can_edit=can_edit,
-        **stats,
+        addable=addable,
+        hosts=hosts,
     )
 
 
@@ -281,7 +226,8 @@ async def create_dashboard():
             db.add(dash)
             await db.flush()
             dash_id = dash.id
-    return redirect(f"/d/{dash_id}/edit")
+    # Open the new (empty) dashboard straight into edit mode to add widgets.
+    return redirect(f"/d/{dash_id}?edit=1")
 
 
 @dashboard_bp.post("/dashboards/<int:dash_id>/rename")
@@ -362,26 +308,12 @@ async def delete_dashboard(dash_id: int):
     return redirect("/dashboards/")
 
 
-# ── Dashboard edit ────────────────────────────────────────────────────────────
-
-@dashboard_bp.get("/d/<int:dash_id>/edit")
-@require_role(UserRole.viewer)
-async def edit(dash_id: int):
-    user_id   = current_user_id()
-    user_role = session.get("user_role", "viewer")
-    async with current_app.db_sessionmaker() as db:
-        dash, resolved, addable = await _get_panels_data(db, dash_id)
-    if dash is None or not _can_edit(dash, user_id, user_role):
-        abort(403)
-    return await render_template(
-        "dashboard/edit.html",
-        dashboard=dash, widgets=resolved, addable=addable,
-    )
-
+# ── Dashboard edit (in-place over the live dashboard) ─────────────────────────
 
 @dashboard_bp.post("/d/<int:dash_id>/edit/add/<widget_key>")
 @require_role(UserRole.viewer)
 async def add_widget(dash_id: int, widget_key: str):
+    """Add a widget and return its single grid-item HTML for in-place insertion."""
     user_id   = current_user_id()
     user_role = session.get("user_role", "viewer")
     defn = WIDGET_REGISTRY.get(widget_key)
@@ -409,12 +341,26 @@ async def add_widget(dash_id: int, widget_key: str):
             if dash is None or not _can_edit(dash, user_id, user_role):
                 abort(403)
             widgets = await _get_widgets(db, dash_id)
-            next_pos = max((w.position for w in widgets), default=-1) + 1
-            db.add(DashboardWidget(
-                dashboard_id=dash_id, widget_key=widget_key, position=next_pos,
+            # Append below everything at full-left; the user then drags/resizes.
+            next_y = max((w.grid_y + w.grid_h for w in widgets), default=0)
+            new = DashboardWidget(
+                dashboard_id=dash_id, widget_key=widget_key,
+                grid_x=0, grid_y=next_y, grid_w=4, grid_h=4,
                 title=title, config_json=config_json,
-            ))
-    return await _panels_response(dash_id)
+            )
+            db.add(new)
+            await db.flush()
+            new_id = new.id
+        new = (await db.execute(
+            select(DashboardWidget).where(DashboardWidget.id == new_id))).scalar_one()
+        resolved = _resolve_widgets([new])
+    if not resolved:
+        return ("", 204)
+    poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
+    return await render_template(
+        "dashboard/_grid_item.html",
+        item=resolved[0], poll_interval=poll_interval, can_edit=True,
+    )
 
 
 @dashboard_bp.post("/d/<int:dash_id>/edit/remove/<int:widget_id>")
@@ -434,30 +380,40 @@ async def remove_widget(dash_id: int, widget_id: int):
             widget = result.scalar_one_or_none()
             if widget and widget.dashboard_id == dash_id:
                 await db.delete(widget)
-                await _normalise_positions(db, dash_id)
-    return await _panels_response(dash_id)
+    # Client removes the DOM/grid item itself; nothing to render back.
+    return ("", 204)
 
 
-@dashboard_bp.post("/d/<int:dash_id>/edit/reorder")
+@dashboard_bp.post("/d/<int:dash_id>/edit/layout")
 @require_role(UserRole.viewer)
-async def reorder_widgets(dash_id: int):
+async def save_layout(dash_id: int):
+    """Persist the drag-resize grid (Gridstack) — one {id,x,y,w,h} per widget."""
     user_id   = current_user_id()
     user_role = session.get("user_role", "viewer")
     data = await request.get_json(silent=True)
-    if not data or not isinstance(data.get("order"), list):
+    if not data or not isinstance(data.get("items"), list):
         abort(400)
-    order: list[int] = data["order"]
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
             result = await db.execute(select(Dashboard).where(Dashboard.id == dash_id))
             dash   = result.scalar_one_or_none()
             if dash is None or not _can_edit(dash, user_id, user_role):
                 abort(403)
-            widgets = await _get_widgets(db, dash_id)
-            widget_map = {w.id: w for w in widgets}
-            for pos, wid in enumerate(order):
-                if wid in widget_map:
-                    widget_map[wid].position = pos
+            widget_map = {w.id: w for w in await _get_widgets(db, dash_id)}
+            for it in data["items"]:
+                if not isinstance(it, dict):
+                    continue
+                w = widget_map.get(it.get("id"))
+                if w is None:
+                    continue
+                try:
+                    # Clamp to the 12-col grid; widths can't exceed it.
+                    w.grid_x = max(0, min(11, int(it["x"])))
+                    w.grid_w = max(1, min(12, int(it["w"])))
+                    w.grid_y = max(0, int(it["y"]))
+                    w.grid_h = max(1, int(it["h"]))
+                except (KeyError, TypeError, ValueError):
+                    continue
     return ("", 204)
 
 

steward/dns/routes.py

@@ -1,52 +0,0 @@
-from __future__ import annotations
-from quart import Blueprint, current_app, render_template
-from sqlalchemy import select, func
-from steward.auth.middleware import require_role
-from steward.models.users import UserRole
-from steward.models.hosts import Host
-from steward.models.monitors import DnsResult
-
-dns_bp = Blueprint("dns", __name__, url_prefix="/dns")
-
-
-async def _latest_dns(db, host_ids: list[str]) -> dict:
-    if not host_ids:
-        return {}
-    subq = (
-        select(DnsResult.host_id, func.max(DnsResult.resolved_at).label("max_at"))
-        .where(DnsResult.host_id.in_(host_ids))
-        .group_by(DnsResult.host_id)
-        .subquery()
-    )
-    pr = await db.execute(
-        select(DnsResult).join(
-            subq,
-            (DnsResult.host_id == subq.c.host_id) & (DnsResult.resolved_at == subq.c.max_at),
-        )
-    )
-    return {r.host_id: r for r in pr.scalars()}
-
-
-@dns_bp.get("/")
-@require_role(UserRole.viewer)
-async def index():
-    async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(Host).where(Host.dns_enabled.is_(True)).order_by(Host.name)
-        )
-        hosts = result.scalars().all()
-        latest = await _latest_dns(db, [h.id for h in hosts])
-    poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
-    return await render_template("dns/index.html", hosts=hosts, latest=latest, poll_interval=poll_interval)
-
-
-@dns_bp.get("/rows")
-@require_role(UserRole.viewer)
-async def rows():
-    async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(Host).where(Host.dns_enabled.is_(True)).order_by(Host.name)
-        )
-        hosts = result.scalars().all()
-        latest = await _latest_dns(db, [h.id for h in hosts])
-    return await render_template("dns/rows.html", hosts=hosts, latest=latest)

steward/hosts/routes.py

@@ -8,9 +8,10 @@ from steward.ansible import executor, sources as ansible_src
 from steward.auth.middleware import require_role
 from steward.core.audit import log_audit
 from steward.models.ansible import AnsibleRun, AnsibleRunStatus
-from steward.models.hosts import Host, ProbeType
-from steward.models.monitors import PingResult, DnsResult, PingStatus
+from steward.models.hosts import Host
+from steward.models.monitors import Monitor, MonitorResult
 from steward.models.users import UserRole
+from steward.core.status import _last_n_results, _uptime_by_monitor, _heartbeat_title
 
 hosts_bp = Blueprint("hosts", __name__, url_prefix="/hosts")
 
@@ -23,7 +24,7 @@ def _ansible_source_names() -> list[str]:
 
 
 async def _compute_uptime(db) -> dict[str, dict]:
-    """Return per-host uptime % for 24h, 7d, 30d windows.
+    """Per-host uptime % for 24h/7d/30d, aggregated over the host's monitors.
 
     Returns {host_id: {"24h": float|None, "7d": float|None, "30d": float|None}}.
     """
@@ -34,25 +35,23 @@ async def _compute_uptime(db) -> dict[str, dict]:
 
     result = await db.execute(
         select(
-            PingResult.host_id,
-            # 30d window — all rows in query qualify
-            func.count(PingResult.id).label("total_30d"),
-            func.sum(case((PingResult.status == PingStatus.up, 1), else_=0)).label("up_30d"),
-            # 7d sub-window
-            func.sum(case((PingResult.probed_at >= cutoff_7d, 1), else_=0)).label("total_7d"),
+            Monitor.host_id,
+            func.count(MonitorResult.id).label("total_30d"),
+            func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up_30d"),
+            func.sum(case((MonitorResult.checked_at >= cutoff_7d, 1), else_=0)).label("total_7d"),
             func.sum(case(
-                (and_(PingResult.probed_at >= cutoff_7d, PingResult.status == PingStatus.up), 1),
+                (and_(MonitorResult.checked_at >= cutoff_7d, MonitorResult.is_up.is_(True)), 1),
                 else_=0,
             )).label("up_7d"),
-            # 24h sub-window
-            func.sum(case((PingResult.probed_at >= cutoff_24h, 1), else_=0)).label("total_24h"),
+            func.sum(case((MonitorResult.checked_at >= cutoff_24h, 1), else_=0)).label("total_24h"),
             func.sum(case(
-                (and_(PingResult.probed_at >= cutoff_24h, PingResult.status == PingStatus.up), 1),
+                (and_(MonitorResult.checked_at >= cutoff_24h, MonitorResult.is_up.is_(True)), 1),
                 else_=0,
             )).label("up_24h"),
         )
-        .where(PingResult.probed_at >= cutoff_30d)
-        .group_by(PingResult.host_id)
+        .join(Monitor, Monitor.id == MonitorResult.monitor_id)
+        .where(Monitor.host_id.isnot(None), MonitorResult.checked_at >= cutoff_30d)
+        .group_by(Monitor.host_id)
     )
 
     def _pct(up, total):
@@ -68,57 +67,253 @@ async def _compute_uptime(db) -> dict[str, dict]:
     return stats
 
 
+async def _host_status(db, host_ids: list[str]) -> dict[str, dict]:
+    """Per-host status rollup from the latest result of each linked monitor.
+
+    {host_id: {"state": "up"|"down"|"pending", "latency_ms": float|None,
+               "count": int}}. State is down if ANY linked monitor's latest
+    result is down (worst-case), up if at least one is up, else pending.
+    """
+    if not host_ids:
+        return {}
+    rn = func.row_number().over(
+        partition_by=MonitorResult.monitor_id, order_by=MonitorResult.checked_at.desc()
+    ).label("rn")
+    subq = (
+        select(MonitorResult.id, rn)
+        .join(Monitor, Monitor.id == MonitorResult.monitor_id)
+        .where(Monitor.host_id.in_(host_ids), Monitor.enabled.is_(True))
+        .subquery()
+    )
+    rows = (await db.execute(
+        select(MonitorResult, Monitor.host_id)
+        .join(subq, MonitorResult.id == subq.c.id)
+        .join(Monitor, Monitor.id == MonitorResult.monitor_id)
+        .where(subq.c.rn == 1)
+    )).all()
+    out: dict[str, dict] = {}
+    for res, host_id in rows:
+        d = out.setdefault(host_id, {"state": "pending", "latency_ms": None, "count": 0})
+        d["count"] += 1
+        if not res.is_up:
+            d["state"] = "down"
+        elif d["state"] != "down":
+            d["state"] = "up"
+        if res.is_up and res.response_ms is not None:
+            d["latency_ms"] = (res.response_ms if d["latency_ms"] is None
+                               else min(d["latency_ms"], res.response_ms))
+    return out
+
+
+async def _host_monitors(db, host_id: str) -> list[dict]:
+    """Display rows for the monitors linked to one host (host hub section)."""
+    monitors = (await db.execute(
+        select(Monitor).where(Monitor.host_id == host_id).order_by(Monitor.name)
+    )).scalars().all()
+    ids = [m.id for m in monitors]
+    recent = await _last_n_results(db, ids)
+    uptime = await _uptime_by_monitor(db, ids)
+    data = []
+    for m in monitors:
+        mrows = recent.get(m.id, [])
+        latest = mrows[-1] if mrows else None
+        data.append({
+            "monitor": m, "latest": latest,
+            "heartbeat": [{"state": "up" if r.is_up else "down",
+                           "title": _heartbeat_title(m.type, r)} for r in mrows],
+            "uptime_pct": uptime.get(m.id, {}).get("24h"),
+        })
+    return data
+
+
+async def _agent_metrics_by_host(db, host_names: list[str]) -> dict[str, dict]:
+    """Latest agent cpu/mem per host name from the generic PluginMetric table.
+
+    Read directly (PluginMetric is a core model) so the Hosts list can show an
+    at-a-glance agent column without importing the host_agent plugin.
+    """
+    from steward.models.metrics import PluginMetric
+    if not host_names:
+        return {}
+    wanted = ("cpu_pct", "mem_used_pct")
+    subq = (
+        select(
+            PluginMetric.resource_name, PluginMetric.metric_name,
+            func.max(PluginMetric.recorded_at).label("m"),
+        )
+        .where(
+            PluginMetric.source_module == "host_agent",
+            PluginMetric.metric_name.in_(wanted),
+            PluginMetric.resource_name.in_(host_names),
+        )
+        .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
+    ).subquery()
+    rows = (await db.execute(
+        select(PluginMetric).join(
+            subq,
+            (PluginMetric.resource_name == subq.c.resource_name)
+            & (PluginMetric.metric_name == subq.c.metric_name)
+            & (PluginMetric.recorded_at == subq.c.m),
+        )
+    )).scalars().all()
+    out: dict[str, dict] = {}
+    for r in rows:
+        out.setdefault(r.resource_name, {})[r.metric_name] = r.value
+    return out
+
+
+async def _agent_overview_by_host(db, host_names: list[str]) -> dict[str, dict]:
+    """Per-host agent glance (cpu/mem/root-disk + freshness) from PluginMetric.
+
+    Core-safe (no host_agent import). Freshness compares the latest host-level
+    sample against the plugin's stale window (config), defaulting to 180s.
+    """
+    from steward.models.metrics import PluginMetric
+    if not host_names:
+        return {}
+    now = datetime.now(timezone.utc)
+    stale_after = int(
+        (current_app.config.get("PLUGINS", {}).get("host_agent", {}) or {})
+        .get("stale_after_seconds", 180))
+    resources = list(host_names) + [h + ":/" for h in host_names]
+    subq = (
+        select(
+            PluginMetric.resource_name, PluginMetric.metric_name,
+            func.max(PluginMetric.recorded_at).label("m"),
+        )
+        .where(
+            PluginMetric.source_module == "host_agent",
+            PluginMetric.resource_name.in_(resources),
+            PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "disk_used_pct")),
+        )
+        .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
+    ).subquery()
+    rows = (await db.execute(
+        select(PluginMetric).join(
+            subq,
+            (PluginMetric.resource_name == subq.c.resource_name)
+            & (PluginMetric.metric_name == subq.c.metric_name)
+            & (PluginMetric.recorded_at == subq.c.m),
+        ).where(PluginMetric.source_module == "host_agent")
+    )).scalars().all()
+    out: dict[str, dict] = {}
+    for r in rows:
+        if r.resource_name.endswith(":/") and r.metric_name == "disk_used_pct":
+            out.setdefault(r.resource_name[:-2], {})["disk_root"] = r.value
+        elif r.resource_name in host_names:
+            d = out.setdefault(r.resource_name, {})
+            d[r.metric_name] = r.value
+            if d.get("_ts") is None or r.recorded_at > d["_ts"]:
+                d["_ts"] = r.recorded_at
+    for d in out.values():
+        ts = d.pop("_ts", None)
+        d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
+    return out
+
+
+async def _agent_cpu_sparks_by_host(db, host_names: list[str], hours: int = 1) -> dict[str, str]:
+    """{host_name: inline-SVG cpu sparkline} over the last `hours` (core-safe).
+
+    A tiny at-a-glance trend in each row — the host-view sparkline, on the widget.
+    """
+    from steward.models.metrics import PluginMetric
+    from steward.core.status import sparkline_svg
+    if not host_names:
+        return {}
+    cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
+    rows = (await db.execute(
+        select(PluginMetric.resource_name, PluginMetric.value, PluginMetric.recorded_at)
+        .where(
+            PluginMetric.source_module == "host_agent",
+            PluginMetric.metric_name == "cpu_pct",
+            PluginMetric.resource_name.in_(host_names),
+            PluginMetric.recorded_at >= cutoff,
+        )
+        .order_by(PluginMetric.recorded_at)
+    )).all()
+    by_host: dict[str, list[float]] = {}
+    for r in rows:
+        by_host.setdefault(r.resource_name, []).append(r.value)
+    # Amber line to read as "CPU"; only draw when there are a couple of points.
+    return {
+        name: sparkline_svg(vals[-40:], width=64, height=18, stroke="#c8a840")
+        for name, vals in by_host.items() if len(vals) >= 2
+    }
+
+
+@hosts_bp.get("/overview/widget")
+@require_role(UserRole.viewer)
+async def overview_widget():
+    """Dashboard widget: unified per-host monitor + agent glance, linking to the hub."""
+    async with current_app.db_sessionmaker() as db:
+        hosts = (await db.execute(select(Host).order_by(Host.name))).scalars().all()
+        host_status = await _host_status(db, [h.id for h in hosts])
+        uptime = await _compute_uptime(db)
+        host_names = [h.name for h in hosts]
+        agent = await _agent_overview_by_host(db, host_names)
+        cpu_sparks = await _agent_cpu_sparks_by_host(db, host_names)
+
+    return await render_template(
+        "hosts/overview_widget.html",
+        hosts=hosts, host_status=host_status,
+        uptime=uptime, agent=agent, cpu_sparks=cpu_sparks,
+    )
+
+
 @hosts_bp.get("/")
 @require_role(UserRole.viewer)
 async def list_hosts():
     async with current_app.db_sessionmaker() as db:
         result = await db.execute(select(Host).order_by(Host.name))
         hosts = result.scalars().all()
-
-        # Latest ping result per host
-        latest_ping_subq = (
-            select(PingResult.host_id, func.max(PingResult.probed_at).label("max_at"))
-            .group_by(PingResult.host_id)
-            .subquery()
-        )
-        pr = await db.execute(
-            select(PingResult).join(
-                latest_ping_subq,
-                (PingResult.host_id == latest_ping_subq.c.host_id)
-                & (PingResult.probed_at == latest_ping_subq.c.max_at),
-            )
-        )
-        latest_pings = {r.host_id: r for r in pr.scalars()}
-
-        # Latest DNS result per host
-        latest_dns_subq = (
-            select(DnsResult.host_id, func.max(DnsResult.resolved_at).label("max_at"))
-            .group_by(DnsResult.host_id)
-            .subquery()
-        )
-        dr = await db.execute(
-            select(DnsResult).join(
-                latest_dns_subq,
-                (DnsResult.host_id == latest_dns_subq.c.host_id)
-                & (DnsResult.resolved_at == latest_dns_subq.c.max_at),
-            )
-        )
-        latest_dns = {r.host_id: r for r in dr.scalars()}
+        host_status = await _host_status(db, [h.id for h in hosts])
         uptime = await _compute_uptime(db)
+        agent_metrics = await _agent_metrics_by_host(db, [h.name for h in hosts])
 
     return await render_template(
         "hosts/list.html",
         hosts=hosts,
-        latest_pings=latest_pings,
-        latest_dns=latest_dns,
+        host_status=host_status,
         uptime=uptime,
+        agent_metrics=agent_metrics,
+    )
+
+
+@hosts_bp.get("/<host_id>")
+@require_role(UserRole.viewer)
+async def host_detail(host_id: str):
+    """The host hub: monitors + agent (embedded fragment) + Ansible, one view."""
+    from steward.models.ansible_inventory import AnsibleTarget
+    from sqlalchemy.orm import selectinload
+
+    async with current_app.db_sessionmaker() as db:
+        host = (await db.execute(
+            select(Host).where(Host.id == host_id))).scalar_one_or_none()
+        if host is None:
+            return "Not found", 404
+        monitors = await _host_monitors(db, host_id)
+        uptime = (await _compute_uptime(db)).get(host_id)
+        linked_target = (await db.execute(
+            select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
+            .options(selectinload(AnsibleTarget.groups)))).scalar_one_or_none()
+        linkable_targets = (await db.execute(
+            select(AnsibleTarget).where(AnsibleTarget.host_id.is_(None))
+            .order_by(AnsibleTarget.name))).scalars().all()
+
+    from steward.models.monitors import MonitorType
+    return await render_template(
+        "hosts/detail.html",
+        host=host, monitors=monitors, uptime=uptime,
+        monitor_types=list(MonitorType),
+        linked_target=linked_target, linkable_targets=linkable_targets,
+        ansible_sources=_ansible_source_names(),
     )
 
 
 @hosts_bp.get("/new")
 @require_role(UserRole.operator)
 async def new_host():
-    return await render_template("hosts/form.html", host=None, probe_types=list(ProbeType))
+    return await render_template("hosts/form.html", host=None)
 
 
 @hosts_bp.post("/")
@@ -128,12 +323,6 @@ async def create_host():
     host = Host(
         name=form["name"].strip(),
         address=form["address"].strip(),
-        probe_type=ProbeType(form.get("probe_type", "tcp")),
-        probe_port=int(form.get("probe_port") or 80),
-        ping_enabled="ping_enabled" in form,
-        dns_enabled="dns_enabled" in form,
-        dns_expected_ip=form.get("dns_expected_ip", "").strip() or None,
-        # poll_interval_seconds not exposed in UI yet; per-host scheduling not implemented
     )
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
@@ -141,44 +330,22 @@ async def create_host():
     await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                     "host.created", entity_type="host", entity_id=host.name,
                     detail={"address": host.address})
-    return redirect(url_for("hosts.list_hosts"))
+    # Land on the new host's hub so monitors/agent/ansible can be set up next.
+    return redirect(url_for("hosts.host_detail", host_id=host.id))
 
 
 @hosts_bp.get("/<host_id>/edit")
 @require_role(UserRole.operator)
 async def edit_host(host_id: str):
-    from steward.models.ansible_inventory import AnsibleTarget
-    from sqlalchemy.orm import selectinload
-
+    # Pure host config (name/address/monitors). Agent, Ansible target linking,
+    # and playbook runs live on the host detail page (the hub), not here.
     async with current_app.db_sessionmaker() as db:
         result = await db.execute(select(Host).where(Host.id == host_id))
         host = result.scalar_one_or_none()
         if host is None:
             return "Not found", 404
 
-        linked_result = await db.execute(
-            select(AnsibleTarget)
-            .where(AnsibleTarget.host_id == host_id)
-            .options(selectinload(AnsibleTarget.groups))
-        )
-        linked_target = linked_result.scalar_one_or_none()
-
-        # Only show targets not yet linked to any host (plus the currently linked one)
-        linkable_result = await db.execute(
-            select(AnsibleTarget)
-            .where(AnsibleTarget.host_id.is_(None))
-            .order_by(AnsibleTarget.name)
-        )
-        linkable_targets = linkable_result.scalars().all()
-
-    return await render_template(
-        "hosts/form.html",
-        host=host,
-        probe_types=list(ProbeType),
-        ansible_sources=_ansible_source_names(),
-        linked_target=linked_target,
-        linkable_targets=linkable_targets,
-    )
+    return await render_template("hosts/form.html", host=host)
 
 
 @hosts_bp.post("/<host_id>")
@@ -193,13 +360,7 @@ async def update_host(host_id: str):
                 return "Not found", 404
             host.name = form["name"].strip()
             host.address = form["address"].strip()
-            host.probe_type = ProbeType(form.get("probe_type", "tcp"))
-            host.probe_port = int(form.get("probe_port") or 80)
-            host.ping_enabled = "ping_enabled" in form
-            host.dns_enabled = "dns_enabled" in form
-            host.dns_expected_ip = form.get("dns_expected_ip", "").strip() or None
-            # poll_interval_seconds not exposed in UI yet
-    return redirect(url_for("hosts.list_hosts"))
+    return redirect(url_for("hosts.host_detail", host_id=host_id))
 
 
 @hosts_bp.post("/<host_id>/ansible-link")
@@ -250,7 +411,7 @@ async def ansible_link(host_id: str):
                     )
                     db.add(new_target)
 
-    return redirect(f"/hosts/{host_id}/edit")
+    return redirect(f"/hosts/{host_id}")
 
 
 @hosts_bp.post("/<host_id>/run-playbook")
@@ -277,24 +438,12 @@ async def run_playbook(host_id: str):
     if playbook not in ansible_src.discover_playbooks(source["path"]):
         return f"Playbook '{playbook}' not found in source '{source_name}'", 404
 
-    # Optional params (no --limit — there's exactly one host).
-    extra_vars: list[str] = []
-    for line in (form.get("extra_vars", "") or "").splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        if "=" not in line:
-            return f"Invalid extra var (expected key=value): {line!r}", 400
-        extra_vars.append(line)
-    params: dict = {}
-    if extra_vars:
-        params["extra_vars"] = extra_vars
-    tags = (form.get("tags", "") or "").strip()
-    if tags:
-        params["tags"] = tags
-    if "check" in form:
-        params["check"] = True
-    params_or_none = params or None
+    # Shared parser: discovered var__ fields → extra_vars_map, secret__ → secret_vars
+    # (unpersisted), plus tags/check. No --limit — there's exactly one host.
+    from steward.ansible.routes import _parse_run_params
+    params_or_none, secret_vars, err = _parse_run_params(form)
+    if err:
+        return err, 400
 
     run_id = str(uuid.uuid4())
     inv_content = ansible_src.host_inventory_content(host)
@@ -315,7 +464,7 @@ async def run_playbook(host_id: str):
         executor.start_run(
             current_app._get_current_object(),  # type: ignore[attr-defined]
             run_id, playbook, f"host: {host.name}", source["path"],
-            params_or_none, inv_content,
+            params_or_none, inv_content, secret_vars=secret_vars,
         )
     )
     task.add_done_callback(
@@ -360,9 +509,7 @@ async def uptime_page():
 async def uptime_widget():
     share_token = request.args.get("s")
     async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(Host).where(Host.ping_enabled.is_(True)).order_by(Host.name)
-        )
+        result = await db.execute(select(Host).order_by(Host.name))
         hosts = result.scalars().all()
         uptime = await _compute_uptime(db)
     return await render_template(