Merge pull request 'Ansible schedule form: structured playbook-variable fields + first-item dropdown defaults' (#2 ) from dev into main

2026-06-30 23:44:04 -04:00
255 changed files with 1871 additions and 17134 deletions
@@ -14,17 +14,6 @@ on:
  # pull_request intentionally absent — push on [dev, main] already fires CI for
  # every dev commit and dev→main PRs. Single-operator repo, no fork PRs.
 # Serialize CI runs per ref so the publish lane never shares the runner's docker
 # daemon with another run. Two publishes racing on one daemon evict each other's
 # freshly-built image mid-push, breaking the post-build tag/push (issue #1093).
 # cancel-in-progress:false is deliberate — rule 46 requires EVERY dev/main push to
 # publish its own immutable :<sha> image, so a superseded run must still run to
 # completion (never cancelled) to emit its SHA. (Forgejo honors `concurrency:` —
 # CI-runner's build workflows use it.)
 concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: false
 jobs:
  # Fast-fail lint lane. ruff is pre-installed in the ci-python image, so this
  # runs with NO dependency install and surfaces lint bounces in seconds.
@@ -126,25 +115,14 @@ jobs:
        run: |
          set -euxo pipefail
          IMAGE=git.fabledsword.com/bvandeusen/steward
-          # The moving tag for this ref: dev→:dev, main→:latest (rule 46). main IS
+          docker build -t "$IMAGE:${{ github.sha }}" .
-          # the production line, so :latest tracks main's tip; there is no :main.
+          docker push "$IMAGE:${{ github.sha }}"
          MOVING=""
          if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
-            MOVING="dev"
+            docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:dev"
            docker push "$IMAGE:dev"
          elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
-            MOVING="latest"
+            docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:latest"
-          fi
+            docker push "$IMAGE:latest"
          # Tag BOTH targets at build time (mirrors CI-runner's build workflows):
          # one `docker build -t :<sha> -t :<moving>` points both tags at the image
          # atomically, so we never run a post-push `docker tag` of an image a
          # concurrent run could have evicted from the shared daemon (issue #1093).
          if [ -n "$MOVING" ]; then
            docker build -t "$IMAGE:${{ github.sha }}" -t "$IMAGE:$MOVING" .
            docker push "$IMAGE:${{ github.sha }}"
            docker push "$IMAGE:$MOVING"
          else
            docker build -t "$IMAGE:${{ github.sha }}" .
            docker push "$IMAGE:${{ github.sha }}"
          fi
      - name: Prune dangling layers
        if: always()
@@ -19,11 +19,10 @@ WORKDIR /app
 COPY pyproject.toml .
 COPY steward/ steward/
-# Bundle the extras whose features ship in the image: [ansible] for the
+# Bundle the extras whose first-party plugins ship in the image: [ansible] for
-# playbook runner, [snmp] (pysnmp) for the SNMP poller, [ldap] (ldap3, pure
+# the playbook runner, [snmp] (pysnmp) for the SNMP poller. Without them those
-# Python) for LDAP sign-in under Settings -> Auth. Without them those features
+# bundled plugins load but silently no-op for want of their runtime dependency.
-# load but fail for want of their runtime dependency.
+RUN pip install --no-cache-dir '.[ansible,snmp]'
 RUN pip install --no-cache-dir '.[ansible,snmp,ldap]'
 COPY alembic.ini .
 # First-party plugins ship inside the image (bundled root at /app/plugins).
@@ -46,7 +46,7 @@ git.fabledsword.com/bvandeusen/ci-python:3.14
 - Steward has **no frontend and no Redis/Celery** (no external workers), so there
  is no frontend-build lane and the only service container is Postgres.
- Integration uses Fabled-Git Actions `services:` + a socket-discovered bridge IP
+- Integration uses Forgejo Actions `services:` + a socket-discovered bridge IP
  because `act_runner` (swarm-runner v0.6+) puts services on the default bridge
  with no embedded DNS. FabledCurator's `ci.yml` is the canonical example of the
  pattern; Steward's is the same shape minus Redis and minus sharding.
@@ -13,14 +13,6 @@
 database:
  url: "postgresql+asyncpg://steward:password@localhost/steward"
  # Optional: seconds to wait at startup for the database to become reachable
  # before giving up (default: 60). Steward retries with backoff rather than
  # crashing when the DB is merely slow to arrive — replaying WAL after an
  # unclean shutdown, or container DNS not yet resolving after a host reboot.
  # Raise it if your database is routinely slower than this to accept
  # connections. Env var: STEWARD_DB_CONNECT_TIMEOUT
  # connect_timeout: 60
 # Optional: override the auto-generated secret key.
 # If not set, a key is auto-generated on first run and saved to /data/secret.key.
 # secret_key: "change-me-to-a-random-string"
@@ -44,30 +44,6 @@ All routes are Quart Blueprints registered in `app.py`:
 Plugin blueprints are mounted automatically by `load_plugins()` using the plugin directory name as the URL prefix.
 `/ai-access/` (`ai_access_bp`, `steward/ai_access/routes.py`) is where a signed-in user manages their API tokens.
 ### `/mcp`: outside the blueprint pipeline
 `mount_mcp(app)` (`steward/mcp/server.py`) wraps `app.asgi_app`. A request to
 `/mcp` is handed to the MCP SDK's streamable-HTTP app before Quart sees it, so
 no blueprint, session cookie or `before_request` hook runs for it. Each
 request then goes through these steps:
 1. **Database gate.** If the database is unreachable, the request gets a 503
   with a readable message.
 2. **Bearer resolution.** `api_tokens.resolve_token()` turns the token into a
   `Caller` (user, effective role, token id). A missing, revoked or expired
   token gets a 401.
 3. **Caller context.** The `Caller` and the app are set in ContextVars
   (`steward/mcp/_context.py`). Tools read them through `current_caller()` and
   `mcp_app()`, because Quart's `current_app` and `session` do not exist there.
 4. **Role gate.** `StewardMCPServer.refusal()` hides and refuses tools above
   the caller's role, and plugin tools whose plugin is disabled.
 The SDK's session manager runs in Quart's `before_serving`/`after_serving`
 lifecycle. The server is stateless: no MCP session outlives a request. See
 [MCP server](core/mcp.md) for the security model and the tools.
 ---
 ## Scheduler
@@ -1,132 +0,0 @@
 # MCP server
 Steward serves the [Model Context Protocol](https://modelcontextprotocol.io) at
 `/mcp`. An AI client connected to it can look into everything Steward watches
 (hosts, monitors, alerts, metrics, containers, playbook runs) and, with a
 high enough role, change it. A token's role sets that ceiling.
 **Source:** `steward/mcp/`. The tools live in `steward/mcp/tools/`, and a
 plugin's tools live in `plugins/<name>/mcp_tools.py`.
 **Transport:** streamable HTTP, stateless. Every request carries its own
 token, so a client survives a Steward restart without reconnecting.
 ---
 ## Connecting
 1. In Steward, open **AI access** and create a token. Pick the lowest role
   that does the job.
 2. Copy the token (it is shown once) and use one of the recipes on that
   page. They carry the real endpoint URL and the new token. For Claude Code:
   ```sh
   claude mcp add --transport http steward https://steward.example/mcp \
     --header "Authorization: Bearer stw_…"
   ```
 3. Ask the client to call `whoami`. It answers with the user, the effective
   role and what that role allows.
 Claude Desktop connects through `mcp-remote`. Open WebUI and other clients
 need an MCP server with the **Streamable HTTP** transport and bearer
 authentication. The AI access page shows the recipes for both.
 ### Behind a reverse proxy
 - The URL on the AI access page is **Settings → General → Public base URL**
  plus `/mcp`. If that setting is empty, the URL is built from the request's
  host instead.
 - The proxy must pass the `Authorization` header through unchanged.
 - It must not buffer responses. Replies may arrive as server-sent events
  (`text/event-stream`). For nginx, set `proxy_buffering off;` on the
  location.
 - Steward does no Host-header (DNS-rebinding) check on `/mcp`, because the
  bearer token is the security boundary. Serve it over HTTPS.
 ---
 ## Security model
 | Rule | How it is enforced |
 |---|---|
 | Bearer tokens only, no OAuth | `steward/core/api_tokens.py`. Only a hash is stored, and the token is shown once. |
 | A token never acts above its user | Effective role = min(token role, user's **current** role). Lowering a user lowers every token they hold; disabling the user stops them all. |
 | Default deny | A tool is reachable only if it was registered with a role (`register_tool(fn, role=..., read_only=...)`). `tools/list` shows a caller only the tools their role allows. A call above the role is refused, and the refusal names the role it needs. |
 | Plugin tools follow the plugin | A disabled plugin's tools are hidden and refused. |
 | Dry run first | `run_playbook`, `create_schedule` and `set_alert_action` default to Ansible check mode (`--check --diff`). Changes are applied only when the caller passes `check=false`. A playbook its author flagged `# steward:confirm: true` also needs `confirmed=true`. `prune_docker` previews unless `check=false`. |
 | Secrets stay put | Playbook variables Steward detects as secret reach the run but are never stored, audited or returned. Schedules and alert actions refuse them, since they would have to store them. Inventory variables are listed by name only, and HTTP monitor header values are redacted. |
 | Every change is audited | Each write records an audit event as `<user> via MCP`, with the token id in its detail. `list_audit_events(user="<user>")` includes those events. |
 | Unknown arguments are refused | A misspelt argument is rejected with a "did you mean", rather than silently dropped. |
 Tokens never mint or revoke tokens; that happens only on the AI access
 page, in a browser session.
 ---
 ## Tools by role
 Each role includes the tools of the roles above it in this list.
 ### Viewer: read everything
 | Area | Tools |
 |---|---|
 | Start here | `whoami`, `get_overview` (what is wrong now, ranked, each finding with the call that shows its evidence) |
 | Hosts and metrics | `list_hosts`, `get_host`, `get_host_metrics`, `query_metrics` |
 | Monitors | `list_monitors`, `get_monitor` |
 | Alerts | `list_alerts`, `get_alert_history`, `list_alert_rules`, `get_alert_rule_options`, `list_maintenance_windows` |
 | Automation | `list_playbooks`, `get_playbook`, `list_inventory`, `list_ansible_runs`, `get_ansible_run`, `wait_for_run`, `list_schedules` |
 | Audit | `list_audit_events` |
 | Docker plugin | `list_containers`, `get_container`, `list_swarm_services`, `search_container_logs`, `get_docker_disk_usage` |
 | Host agent plugin | `list_agents` |
 ### Operator: run playbooks, change monitoring
 | Area | Tools |
 |---|---|
 | Automation | `run_playbook`, `cancel_ansible_run`, `create_schedule`, `update_schedule`, `delete_schedule` |
 | Monitors | `create_monitor`, `update_monitor`, `set_monitor_enabled` |
 | Alerts | `create_alert_rule`, `update_alert_rule`, `acknowledge_alert`, `create_maintenance_window`, `end_maintenance_window` |
 ### Admin: destructive and remediation
 `set_alert_action` (the playbook a rule runs when it fires), `delete_monitor`,
 `delete_alert_rule`, `prune_docker`.
 The client sees each tool's full contract: arguments, what an empty value
 means, and what comes back. That contract is the tool's docstring, so read the
 docstrings in `steward/mcp/tools/` for the detail.
 ---
 ## Response conventions
 Every tool follows these conventions (`steward/mcp/tools/_shape.py`):
 - **Times** are ISO-8601 UTC with a `Z`. Anything with a "when" also carries
  `age_s`, so the client never does clock arithmetic.
 - **Lists are bounded.** A list response carries `total` and `truncated`. When
  it was cut, `more` names the argument that fetches the rest. Nothing is cut
  silently.
 - **Time series** come back as a summary (min/avg/max/p95/last) plus a series
  of at most 60 points, never raw rows.
 - **Lookup by name or id.** A miss suggests close matches and names the tool
  that lists the valid choices.
 - **Refusals are readable.** Bad input becomes a tool error that says what was
  wrong, why, and what to do instead, and nothing is changed. The tools share
  their validation with the web forms (`monitors/service.py`,
  `alerts/service.py`), so a value the UI refuses is refused here for the same
  reason.
 - **Database outages** return a plain "Steward cannot reach its database"
  rather than a stack trace.
 ---
 ## Adding tools
 Core tools: add an `async def` to a module in `steward/mcp/tools/` and list it
 in that module's `register()` with its minimum role and `read_only`. A plugin
 exposes `register_mcp_tools(register)`, and optionally `mcp_overview(db)` to
 contribute findings to `get_overview`. See
 [Writing a plugin](../plugins/writing-a-plugin.md). A tool that changes
 anything calls `audit_mcp(...)` from `steward/mcp/tools/_audit.py`.
@@ -74,15 +74,8 @@ token = a1b2c3d4...
 interval_seconds = 30
 hostname = myhost              # optional; defaults to uname -n
 mounts = /, /mnt/data          # optional; defaults to all real mounts (excluding tmpfs/devtmpfs/etc.)
 docker_logs_enabled = true     # optional; on by default — per-host container-log kill-switch
 docker_log_exclude = watchtower, noisy-svc   # optional; container names whose logs this host skips collecting
 ```
 Container-log collection is on by default (no config needed). The two keys above
 are the per-host opt-outs; the operator-facing global toggle + exclude live in
 Settings and are enforced server-side (the push model has no channel to command
 an agent), so they don't require touching a host's conf.
 ### Agent internals (function list, not classes)
 - `read_config(path)` — parses the conf file into a dict.
@@ -92,17 +85,10 @@ an agent), so they don't require touching a host's conf.
 - `collect_load()` — reads `/proc/loadavg`, returns `[1m, 5m, 15m]`.
 - `collect_uptime()` — reads `/proc/uptime`, returns seconds since boot (int).
 - `collect_metadata()` — `os.uname()` for kernel + arch, `/etc/os-release` for distro. Called once at startup and cached.
 - `collect_docker_logs(socket, containers, state, exclude)` — (m79) per running
  container, fetches new log lines over the Docker socket with an incremental
  `since` cursor kept per container in `state` (a container's first interval
  seeds from a short tail). Demuxes the multiplexed stream, parses each line's
  RFC3339 timestamp, caps the whole batch at a byte limit (a marker line records
  a truncation; the deferred lines come next interval). Folded into the sample as
  `docker_logs`; omitted when empty.
 - `build_payload()` — assembles a snapshot from all collectors into one dict.
 - `post_payload(url, token, payloads)` — POSTs a list of samples, returns success/failure.
 - `RingBuffer(maxlen=20)` — tiny FIFO wrapper, drops oldest when full.
- `main_loop()` — the 30s loop: collect → try POST → on failure push to buffer + backoff → on success flush buffer. Container logs are stripped from a sample before it's buffered (metrics survive an outage; stale logs are dropped).
+- `main_loop()` — the 30s loop: collect → try POST → on failure push to buffer + backoff → on success flush buffer.
 **Target: ~300 lines total including docstrings.** More than that is a smell that the agent is over-scoping.
@@ -269,7 +255,6 @@ Content-Type: application/json
 - **`metadata` is sent on every POST**, not just on change. Server-side diff detects actual changes and only writes on change. Cost per POST is one dict — negligible. Benefit: server can cleanly detect agent restarts.
 - **Raw bytes, not percentages, for memory and storage.** Percentages are derived server-side. Changing the "what counts as used" math doesn't require re-releasing the agent.
 - **CPU is the one exception** — reported as a percentage because it's inherently a derivative (delta over time), not a snapshot. The agent must sample twice to compute it.
 - **Container logs (m79) ride in the same push** as `docker_logs`: a list of `{container, stream, ts, line}` records — incremental since the previous interval. This pushes the *same direction* as metrics, so batched log history needs no inbound channel (only sub-second live-follow would). The server ingests them into `docker_logs`, enforces the global toggle + exclude list on ingest, and bounds storage with a per-container size+age ring. `docker_logs` is omitted when there's nothing new.
 ### Server expansion into `PluginMetric` rows
@@ -323,61 +323,6 @@ Role hierarchy: `admin > operator > viewer`. Requiring `viewer` grants access to
 ---
 ## Agent Tools (MCP)
 Steward serves an MCP endpoint at `/mcp` that AI clients reach with a personal API token created on the AI access page. A plugin adds tools to it by exporting an optional `register_mcp_tools(register)` from `__init__.py`:
 ```python
 # plugins/myplugin/__init__.py
 def register_mcp_tools(register) -> None:
    from .mcp_tools import list_targets, rescan_target
    from steward.models.users import UserRole
    register(list_targets, role=UserRole.viewer, read_only=True)
    register(rescan_target, role=UserRole.operator, read_only=False)
 ```
 ```python
 # plugins/myplugin/mcp_tools.py
 from steward.mcp._context import current_caller, mcp_app
 async def list_targets(limit: int = 50) -> dict:
    """What this tool is for, when to use it over its siblings, what each
    argument's empty value means, and what comes back. This docstring is the
    tool's description: it is all the agent knows about it."""
    app = mcp_app()                 # Quart's current_app does not exist here
    async with app.db_sessionmaker() as db:
        ...
    return {"targets": [...], "total": n, "truncated": n > limit}
 ```
 - `role` and `read_only` are required. A tool is listed and callable only for callers whose role reaches `role`, and only while the plugin is enabled.
 - Tools run outside Quart's request pipeline: use `mcp_app()` and `current_caller()` from `steward.mcp._context`, never `current_app` or `session`.
 - Refuse by raising `ValueError("what was wrong, and what to do instead")`. The agent reads that text. Any other exception reaches it only as a generic failure.
 - Tool names are global across core and every plugin; a name already taken fails registration and the plugin contributes no tools (logged).
 - Hot-reloading the plugin replaces its tools.
 - Follow the response conventions the core tools use (`steward/mcp/tools/_shape.py`): times via `iso()` plus `age_s()`, lists through `bounded()` so a cut is never silent, series through `summarize()`, and lookups by name or id through `pick()`.
 ### Contributing to get_overview
 `get_overview` is the agent's first call: what is wrong right now, ranked. A plugin adds its own problems to it by exporting an optional `mcp_overview(db)` that returns findings built with `steward.mcp.tools.overview.finding`:
 ```python
 # plugins/myplugin/__init__.py
 async def mcp_overview(db) -> list[dict]:
    from steward.mcp.tools.overview import finding
    return [
        finding("warning", "target_unreachable", t.name, f"{t.name} has not answered for {t.misses} scans",
                since=t.last_ok_at, ids={"target_id": t.id}, next=f'get_target(target="{t.id}")')
        for t in await _unreachable_targets(db)
    ]
 ```
 - `severity` is `"critical"` (something is down or about to be) or `"warning"` (degraded, or heading that way). Report problems only; a healthy plugin returns `[]`.
 - It is called only while the plugin is enabled, with its own session. If it raises, the overview still returns, and names the plugin under `unavailable`.
 ---
 ## Publishing to the Catalog
 The official plugin catalog is hosted at `https://git.fabledsword.com/bvandeusen/Steward-plugins`. Anyone can submit a plugin by opening a pull request — first-party and third-party plugins are treated identically by the catalog system.
@@ -468,7 +413,6 @@ Hot-reload works reliably for brand-new plugin installs. Updates to already-acti
 - [ ] Revision IDs prefixed with plugin name
 - [ ] `record_metric()` called inside `session.begin()`
 - [ ] Routes use `@require_role` decorator
 - [ ] MCP tools (if any) registered via `register_mcp_tools(register)` with a role and `read_only`
 - [ ] Templates namespaced under `templates/myplugin/`
 - [ ] Plugin enabled in app settings
 - [ ] `index.yaml` entry added with `download_url` and `checksum_sha256`
@@ -136,25 +136,3 @@ Quick reference for where key functions, models, and entry points live in the co
 | `steward/migrations/versions/` | Core schema (hosts, users, monitors, alerts, app_settings) |
 | `plugins/traefik/migrations/versions/` | `traefik_metrics` table |
 | `alembic.ini` | Alembic config; `version_locations` lists all migration directories |
 ---
 ## MCP server and API tokens
 | What | File | Function / Class |
 |---|---|---|
 | Mount `/mcp` (ASGI dispatch, bearer auth, DB gate) | `steward/mcp/server.py` | `mount_mcp(app)` |
 | Server, role registry, refusals | `steward/mcp/server.py` | `StewardMCPServer`, `register_tool()`, `refusal()` |
 | Plugin tools and overview findings | `steward/mcp/server.py` | `register_plugin(name, module)` |
 | Caller and app inside a tool | `steward/mcp/_context.py` | `current_caller()`, `mcp_app()` |
 | Core tool modules | `steward/mcp/tools/` | `register_all()`; one module per area |
 | Response conventions | `steward/mcp/tools/_shape.py` | `iso`, `age_s`, `bounded`, `lookback`, `summarize`, `pick` |
 | Audit a write as "<user> via MCP" | `steward/mcp/tools/_audit.py` | `audit_mcp()` |
 | "What is wrong now" | `steward/mcp/tools/overview.py` | `get_overview()` |
 | Playbooks, runs, schedules, alert actions | `steward/mcp/tools/automation.py` | `run_playbook()`, `wait_for_run()`, … |
 | Monitor validation (forms and tools) | `steward/monitors/service.py` | `monitor_fields()`, `build_config()` |
 | Alert rule / window validation | `steward/alerts/service.py` | `rule_fields()`, `window_fields()`, `acknowledge()` |
 | Docker and host-agent tools | `plugins/docker/mcp_tools.py`, `plugins/host_agent/mcp_tools.py` | `register_tools()`, `mcp_overview()` |
 | Token model | `steward/models/api_tokens.py` | `ApiToken` |
 | Mint, resolve, revoke tokens | `steward/core/api_tokens.py` | `create_token()`, `resolve_token()`, `revoke_token()` |
 | Token management page and connect card | `steward/ai_access/routes.py`, `steward/templates/ai_access/` | `ai_access_bp` |
@@ -49,23 +49,6 @@ def get_blueprint():
    return docker_bp
 def register_mcp_tools(register) -> None:
    from .mcp_tools import register_tools
    register_tools(register)
 async def mcp_overview(db) -> list[dict]:
    from .mcp_tools import mcp_overview as findings
    return await findings(db)
 def get_nav() -> list[dict]:
    # Surfaces in the sidebar's Infrastructure group (fleet-wide container view).
    return [{"label": "Docker", "href": "/plugins/docker/"}]
 def get_alert_metrics() -> dict[str, list[str]]:
    """What ingest.py records per container (resource "<host>/<container>"):
    restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 /
    0.0 from the container's HEALTHCHECK, only for containers that define one."""
    return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy"]}
@@ -92,55 +92,6 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
    return events
 def _log_rows(log_batches, host_id: str):
    """Pure: flatten the agent's log batches → docker_logs row kwargs (no DB).
    `log_batches` is a list of (recorded_at, records) where each record is
    {"container", "stream", "ts", "line"}. A line's own Docker timestamp wins;
    recorded_at is the fallback when the agent couldn't parse one. Drops the
    agent's advisory truncation marker (container "_steward" — it signals a
    deferral, not a real container), malformed records, and lineless records;
    normalises an unknown stream to stdout. Unit-testable in isolation.
    """
    for recorded_at, records in log_batches:
        if not isinstance(records, list):
            continue
        for rec in records:
            if not isinstance(rec, dict):
                continue
            name = rec.get("container")
            line = rec.get("line")
            if not name or name == "_steward" or line is None:
                continue
            ts = _parse_started_at(rec.get("ts")) or recorded_at
            stream = rec.get("stream")
            if stream not in ("stdout", "stderr"):
                stream = "stdout"
            yield {"host_id": host_id, "container_name": str(name)[:255],
                   "ts": ts, "stream": stream, "line": str(line)}
 async def _persist_logs(session, host, log_batches) -> None:
    """Append pushed container log lines (one row per line) for this host.
    Time-series / append-only — the per-container size+age ring (retention)
    bounds growth, so a chatty container just keeps a shorter window. The global
    toggle + exclude list (Settings, no restart) are enforced here: this is the
    authoritative drop point, since the push model has no channel to tell an
    agent to stop collecting.
    """
    from steward.core.settings import get_setting
    from .models import DockerLog
    if not await get_setting(session, "docker.logs.enabled"):
        return
    exclude = set(await get_setting(session, "docker.logs.exclude") or ())
    for row in _log_rows(log_batches, host.id):
        if row["container_name"] in exclude:
            continue
        session.add(DockerLog(**row))
 async def _persist_swarm(session, host, swarm: dict) -> None:
    """Upsert this manager's swarm topology; drop rows no longer reported.
@@ -169,7 +120,6 @@ async def _persist_swarm(session, host, swarm: dict) -> None:
        row.running = int(s.get("running") or 0)
        row.image = s.get("image")
        row.placement_json = json.dumps(s.get("placement") or [])
        row.tasks_json = json.dumps(s.get("tasks") or [])
        row.updated_at = now
    stale_services = delete(DockerSwarmService).where(
        DockerSwarmService.host_id == host.id)
@@ -251,8 +201,7 @@ async def _persist_disk(session, host, disk: dict) -> None:
    await session.execute(stale)
-async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
+async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -> None:
                              logs=None) -> None:
    """Upsert containers + time-series + lifecycle events + swarm for one host.
    `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
@@ -262,19 +211,15 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
    alert pipeline, and lifecycle-event derivation. `swarm` is the newest
    sample's swarm object (or None off managers) — persisted when present.
    `disk` is the newest sample's /system/df summary (or None on Docker-less
-    hosts) — persisted when present. `logs` is a list of (recorded_at, records)
+    hosts) — persisted when present.
    log batches (m79) — appended one row per line when present, independent of
    whether this sample also carried container metrics.
    """
-    from steward.core.alerts import MetricBatch
+    from steward.core.alerts import record_metric
    from .models import DockerContainer, DockerEvent, DockerMetric
    if swarm is not None:
        await _persist_swarm(session, host, swarm)
    if disk is not None:
        await _persist_disk(session, host, disk)
    if logs:
        await _persist_logs(session, host, logs)
    if not snapshots:
        return
@@ -313,7 +258,6 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
            ))
    # Current state + alerts from the newest snapshot only.
    batch = MetricBatch()
    for c in latest_containers:
        name = c.get("name")
        if not name:
@@ -351,19 +295,19 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
        # on different hosts don't collide in the metric/alert namespace.
        resource = f"{host.name}/{name}"
        if c.get("status") == "running" and c.get("cpu_pct") is not None:
-            batch.add(
+            await record_metric(
-                source_module="docker",
+                session=session, source_module="docker",
                resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
            )
            if c.get("mem_pct") is not None:
-                batch.add(
+                await record_metric(
-                    source_module="docker",
+                    session=session, source_module="docker",
                    resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
                )
        # Restart count is alertable regardless of state (crash-looping matters
        # most while the container is down/restarting).
-        batch.add(
+        await record_metric(
-            source_module="docker",
+            session=session, source_module="docker",
            resource_name=resource, metric_name="restart_count",
            value=float(c.get("restart_count", 0) or 0),
        )
@@ -372,14 +316,12 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
        # container).
        health = c.get("health")
        if health in ("healthy", "unhealthy", "starting"):
-            batch.add(
+            await record_metric(
-                source_module="docker",
+                session=session, source_module="docker",
                resource_name=resource, metric_name="is_healthy",
                value=1.0 if health == "healthy" else 0.0,
            )
    await batch.record(session)
    # Reap containers that vanished from the host's latest listing. Without this,
    # removed one-shot containers (CI job runners, buildkit builders, codex jobs)
    # accumulate forever as permanent "stopped" rows and inflate every count.
@@ -1,703 +0,0 @@
 """The docker plugin's MCP tools and its get_overview findings.
 Registered through register_mcp_tools() in __init__.py, so they exist only
 while the plugin is enabled. Reads go through service.py, the same functions
 the plugin's pages use.
 """
 from __future__ import annotations
 import json
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from sqlalchemy import select
 from steward.ansible.runner import trigger_run
 from steward.ansible.sources import BUILTIN_SOURCE_NAME
 from steward.mcp._context import current_caller, mcp_app
 from steward.mcp.tools._audit import audit_mcp
 from steward.mcp.tools._shape import (
    age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, summarize,
 )
 from steward.mcp.tools.overview import finding
 from steward.models.ansible_inventory import AnsibleTarget
 from steward.models.hosts import Host
 from steward.models.users import UserRole
 from . import service as docker_svc
 from .dedup import dedup_by_container_id
 from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
 from .swarm_view import build_swarm_services, replaced_tasks, task_time
 PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
 CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
 LOG_LINE_CHARS = 2000
 # ── pure helpers ─────────────────────────────────────────────────────────────
 def container_problems(c, failures_1h: int, failed_24h: bool,
                       crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
    """What is wrong with one container, most severe first; [] when nothing is."""
    status = (c.status or "").lower()
    out = []
    if status == "restarting":
        out.append("restarting")
    if failures_1h >= crash_loop_deaths:
        out.append("crash_looping")
    if status == "running" and (c.health or "") == "unhealthy":
        out.append("unhealthy")
    if c.oom_killed:
        out.append("oom_killed")
    if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h:
        out.append("exited_nonzero")
    return out
 def problem_since(problems: list[str], deaths: list, hour_ago: datetime,
                  unhealthy_at: datetime | None = None) -> datetime | None:
    """When a container's worst problem began; None when the retained events
    cannot say. `deaths` are its die/oom events of the last 24h, newest first.
    A restart or crash loop dates from its first death in the last hour, an
    OOM kill or error exit from the death that ended it, and a failing
    healthcheck from when its health last turned unhealthy.
    """
    if not problems:
        return None
    worst = problems[0]
    if worst in ("restarting", "crash_looping"):
        in_hour = [e.at for e in deaths if e.at >= hour_ago]
        return min(in_hour) if in_hour else (deaths[0].at if deaths else None)
    if worst == "unhealthy":
        return unhealthy_at
    if worst == "oom_killed":
        ooms = [e.at for e in deaths if e.event == "oom"]
        return ooms[0] if ooms else (deaths[0].at if deaths else None)
    return deaths[0].at if deaths else None
 def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
    """Keep both ends of an over-long line and say how much of the middle went."""
    if text is None or len(text) <= limit:
        return text
    keep = limit // 2
    return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
 def container_row(c, host_names: dict, problems: list[str], now: datetime,
                  replaced: bool = False, since: datetime | None = None) -> dict:
    running = (c.status or "").lower() == "running"
    return {
        "container_name": c.name,
        "host_id": c.host_id,
        "host_name": host_names.get(c.host_id),
        "image": c.image,
        "status": c.status,
        "health": c.health,
        "problems": problems,
        # When the worst problem began, from the container's retained events.
        "problem_since": iso(since),
        "problem_age_s": age_s(since, now),
        "exit_code": c.exit_code,
        "oom_killed": bool(c.oom_killed),
        "restart_count": c.restart_count,
        "cpu_pct": rnd(c.cpu_pct),
        "mem_pct": rnd(c.mem_pct),
        "mem_usage_bytes": c.mem_usage_bytes,
        "mem_limit_bytes": c.mem_limit_bytes,
        "started_at": iso(c.started_at),
        "uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
        "compose_project": c.compose_project,
        "swarm_service": c.service_name,
        # A swarm task Swarm has replaced: history, judged on its service.
        "swarm_replaced": replaced if c.service_name else None,
        "reported_at": iso(c.scraped_at),
        "reported_age_s": age_s(c.scraped_at, now),
    }
 _SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
             "oom_killed": "warning", "exited_nonzero": "warning"}
 _SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
         "unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
         "exited_nonzero": "exited with an error"}
 def _from_iso(text: str | None) -> datetime | None:
    return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
 def container_findings(rows: list[dict]) -> list[dict]:
    out = []
    for r in rows:
        if not r["problems"]:
            continue
        worst = r["problems"][0]
        where = f" on {r['host_name']}" if r["host_name"] else ""
        detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
        out.append(finding(
            _SEVERITY[worst], f"container_{worst}", r["container_name"],
            f"container {r['container_name']}{where} {' and '.join(_SAYS[p] for p in r['problems'])}{detail}",
            since=_from_iso(r["problem_since"]),
            ids={"host_id": r["host_id"], "container_name": r["container_name"]},
            next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
    return out
 def task_error_text(err: dict | None) -> str:
    """Swarm's own account of a task's end, as one clause."""
    if not err:
        return ""
    why = err.get("error") or err.get("message") or err.get("state") or "unknown"
    code = f", exit {err['exit_code']}" if err.get("exit_code") not in (None, 0) else ""
    where = f" on {err['node']}" if err.get("node") else ""
    return f"{why}{code}{where}"
 def service_findings(services: list[dict]) -> list[dict]:
    """Swarm services that are failing: crash-looping, under-replicated, or that
    lost a task to an error in the last 24h. Each names Swarm's last task error."""
    out = []
    for s in services:
        replicas = f"{s['running']} of {s['desired']} replicas running"
        last = task_error_text(s.get("last_error"))
        tail = f"; last task error: {last}" if last else ""
        ids = {"service_name": s["name"]}
        since = s.get("trouble_since")
        if s.get("crash_looping"):
            out.append(finding(
                "critical", "swarm_service_crash_looping", s["name"],
                f"swarm service {s['name']} keeps failing: {s['failures_1h']} tasks failed in the "
                f"last hour, {replicas}{tail}", since=since, ids=ids,
                next="list_swarm_services()"))
        elif s["state"] != "healthy":
            out.append(finding(
                "critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
                s["name"], f"swarm service {s['name']} has {replicas}{tail}",
                since=since, ids=ids, next="list_swarm_services()"))
        elif s.get("failures_24h"):
            n = s["failures_24h"]
            out.append(finding(
                "warning", "swarm_task_failed", s["name"],
                f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
                f"24h and is now {replicas}{tail}", since=since, ids=ids,
                next="list_swarm_services()"))
    return out
 # ── loading ──────────────────────────────────────────────────────────────────
 async def _host_names(db) -> dict[str, str]:
    return dict((await db.execute(select(Host.id, Host.name))).all())
 def _crash_loop_deaths() -> int:
    return docker_svc.crash_loop_deaths(mcp_app().config)
 async def _replaced(db, containers) -> set[tuple]:
    services = list((await db.execute(select(DockerSwarmService))).scalars())
    return replaced_tasks(containers, services)
 async def _rows(db, now: datetime) -> list[dict]:
    """Every container with its problems. A swarm task Swarm replaced has none:
    its failure, if it was one, belongs to its service (service_findings)."""
    containers = dedup_by_container_id(await docker_svc.list_containers(db))
    replaced = await _replaced(db, containers)
    failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
    hour_ago = now - timedelta(hours=1)
    host_names = await _host_names(db)
    deaths = _crash_loop_deaths()
    sick = await docker_svc.unhealthy_since(
        db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"])
    rows = []
    for c in containers:
        key = (c.host_id, c.name)
        gone = key in replaced
        events = failures.get(key, [])
        recent = sum(1 for e in events if e.at >= hour_ago)
        problems = [] if gone else container_problems(c, recent, bool(events), deaths)
        since = problem_since(problems, events, hour_ago, sick.get(key))
        rows.append(container_row(c, host_names, problems, now, replaced=gone, since=since))
    return rows
 async def _swarm(db, now: datetime):
    containers = dedup_by_container_id(await docker_svc.list_containers(db))
    services = list((await db.execute(select(DockerSwarmService))).scalars())
    nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
    host_names = await _host_names(db)
    node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
    built = build_swarm_services(containers, services, node_hostname, host_names,
                                 now=now, crash_loop_deaths=_crash_loop_deaths())
    return built["services"], nodes, host_names
 async def mcp_overview(db) -> list[dict]:
    """get_overview findings: failing containers and under-replicated swarm services."""
    now = now_utc()
    rows = await _rows(db, now)
    services, _, _ = await _swarm(db, now)
    return container_findings(rows) + service_findings(services)
 # ── tools ────────────────────────────────────────────────────────────────────
 async def list_containers(host: str = "", status: str = "", problem: str = "",
                          compose_project: str = "", limit: int = 100) -> dict:
    """Docker containers across every host that runs a Steward agent with Docker.
    host: a host's name or id (list_hosts); empty for all hosts.
    status: Docker's state — "running", "exited", "restarting", "paused",
      "created" or "dead"; empty for all.
    problem: "any" for containers with something wrong, or one of
      "restarting", "crash_looping" (died repeatedly within the last hour;
      the count is set under Settings → Thresholds, default 3), "unhealthy"
      (failing its healthcheck), "oom_killed", "exited_nonzero" (died with an
      error in the last 24h); empty for no filter.
    compose_project: only containers from this Compose project.
    limit: default 100, max 500.
    Containers with problems come first. Each row's `problems` lists what is
    wrong, most severe first, and `problem_since` / `problem_age_s` say when
    the worst of them began (null when Steward's retained events cannot say).
    `reported_age_s` is how old the agent's last report is, so a large value
    means the row may be stale.
    A swarm task's container that Swarm has replaced (a rolling update, or a
    retry after a failure) has `swarm_replaced: true` and no problems: Swarm
    restarts a task as a new container, so failures are counted on the
    service — list_swarm_services shows them.
    """
    if problem and problem != "any" and problem not in PROBLEMS:
        raise ValueError(f"problem {problem!r} is not 'any' or one of {', '.join(PROBLEMS)}.")
    limit = clamp_limit(limit, 100, 500)
    now = now_utc()
    async with db_session() as db:
        rows = await _rows(db, now)
        if host:
            hosts = list((await db.execute(select(Host))).scalars())
            h = pick(host, hosts, kind="host", list_tool="list_hosts")
            rows = [r for r in rows if r["host_id"] == h.id]
    if status:
        rows = [r for r in rows if (r["status"] or "").lower() == status.lower()]
    if problem == "any":
        rows = [r for r in rows if r["problems"]]
    elif problem:
        rows = [r for r in rows if problem in r["problems"]]
    if compose_project:
        rows = [r for r in rows if r["compose_project"] == compose_project]
    rows.sort(key=lambda r: (not r["problems"], (r["status"] or "") != "running",
                             (r["host_name"] or "").lower(), r["container_name"]))
    return bounded(rows, limit, key="containers",
                   more="Pass a larger `limit` (max 500), or filter by `host`, `status` or `problem`.")
@dataclass
 class RemovedContainer:
    """A container Docker has removed, known only by its retained logs and events."""
    host_id: str
    name: str
    last_seen: datetime
 _REMOVED_HINT = ("It may have been removed: Swarm replaces a dead task with a new container under "
                 "a new name. list_containers(status=\"exited\") lists stopped containers, and "
                 "search_container_logs(host=...) searches a whole host.")
 async def _pick_container(db, container: str, host: str, *, removed: bool = False):
    """The container named `container`: a live one, else (with `removed`) one
    Docker has removed whose logs or events Steward still holds, returned as a
    RemovedContainer. Exact names beat partial ones, and live beats removed."""
    containers = await docker_svc.list_containers(db)
    host_id = None
    if host:
        hosts = list((await db.execute(select(Host))).scalars())
        host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
        containers = [c for c in containers if c.host_id == host_id]
    want = (container or "").strip()
    if not want:
        raise ValueError("pass a container name; list_containers lists them.")
    live = {(c.host_id, c.name) for c in containers}
    def _one(found, what):
        if len(found) > 1:
            raise ValueError(f"{len(found)} hosts have {what} named {want!r}; pass `host` "
                             f"(one of: {', '.join(sorted(f[0] for f in found))}).")
        return found[0] if found else None
    same = [c for c in containers if c.name == want] or \
           [c for c in containers if c.name.lower() == want.lower()]
    if same:
        return _one([(c.host_id, c) for c in same], "a container")[1]
    if removed:
        gone = [r for r in await docker_svc.retained_names(db, want, exact=True, host_id=host_id)
                if (r[0], r[1]) not in live]
        if gone:
            return RemovedContainer(*_one(gone, "a removed container"))
    # One more chance: a unique container whose name contains it (swarm task
    # names look like "stack_svc.1.k2j…").
    partial = [c for c in containers if want.lower() in c.name.lower()]
    if len(partial) == 1:
        return partial[0]
    gone = []
    if removed and not partial:
        gone = [r for r in await docker_svc.retained_names(db, want, exact=False, host_id=host_id)
                if (r[0], r[1]) not in live]
        if len(gone) == 1:
            return RemovedContainer(*gone[0])
    names = sorted(c.name for c in partial)[:10] + [f"{r[1]} (removed)" for r in gone[:10]]
    hint = f" Containers whose names contain it: {', '.join(names)}." if names else ""
    raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} "
                     f"{'' if names else _REMOVED_HINT + ' '}list_containers lists them.")
 def _removed_note(c: RemovedContainer) -> str:
    return (f"{c.name} no longer exists: Docker removed it (Swarm replaces a failed task with a new "
            f"container). This is what Steward retained; it was last seen {iso(c.last_seen)}.")
 async def get_container(container: str, host: str = "", range: str = "24h", events: int = 20) -> dict:
    """One container in depth: state, resources over time and lifecycle events.
    container: the container's name, as list_containers shows. A unique part of
      a name also works (swarm task names are long).
    host: the host's name or id. Needed only when several hosts run a
      container with this name.
    range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
      max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
      "7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
    events: how many lifecycle events (start, stop, die, oom, health_change)
      to return, newest first (default 20, max 100).
    A container Docker has since removed (a dead Swarm task, a `docker rm`)
    still resolves by its exact name, or by a part of it that matches one
    removed container, while Steward retains its logs or events: the reply
    has `removed: true`, `last_seen_at`, its history and its events, and no
    live state. A live container has `removed: false`.
    """
    since, key = lookback(range)
    events = clamp_limit(events, 20, 100)
    now = now_utc()
    async with db_session() as db:
        c = await _pick_container(db, container, host, removed=True)
        if isinstance(c, RemovedContainer):
            return await _removed_container(db, c, since, key, events, now)
        gone = (c.host_id, c.name) in await _replaced(db, await docker_svc.list_containers(db))
        failures = (await docker_svc.recent_failures_by_container(
            db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
        host_names = await _host_names(db)
        cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
        mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
        evs = await docker_svc.container_events(db, c.host_id, c.name, events)
        sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)])
    hour_ago = now - timedelta(hours=1)
    recent = sum(1 for e in failures if e.at >= hour_ago)
    try:
        ports = json.loads(c.ports_json or "[]")
    except ValueError:
        ports = []
    problems = [] if gone else container_problems(c, recent, bool(failures), _crash_loop_deaths())
    began = problem_since(problems, failures, hour_ago, sick.get((c.host_id, c.name)))
    return {
        **container_row(c, host_names, problems, now, replaced=gone, since=began),
        "removed": False,
        "container_id": c.container_id,
        "ports": ports,
        "net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
        "blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
        "range": key,
        "cpu_pct_history": summarize(cpu),
        "mem_pct_history": summarize(mem),
        "events": _event_rows(evs, now),
        "next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
    }
 def _event_rows(evs, now: datetime) -> list[dict]:
    return [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)}
            for e in evs]
 async def _removed_container(db, c: RemovedContainer, since, key: str, events: int,
                             now: datetime) -> dict:
    """get_container for a container Docker has removed: what Steward kept."""
    host_names = await _host_names(db)
    cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
    mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
    evs = await docker_svc.container_events(db, c.host_id, c.name, events)
    return {
        "container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id),
        "removed": True, "note": _removed_note(c),
        "last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now),
        "range": key,
        "cpu_pct_history": summarize(cpu),
        "mem_pct_history": summarize(mem),
        "events": _event_rows(evs, now),
        "next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")',
    }
 async def list_swarm_services(limit: int = 100) -> dict:
    """Docker Swarm services with their replicas per node, and the swarm's nodes.
    `state` is "healthy" (all desired replicas running), "degraded" (some) or
    "down" (none). `replicas` are the service's current tasks: each names its
    host and container status, and one marked `ghost` runs on a node with no
    Steward agent, so Steward knows only how many tasks are placed there.
    Tasks Swarm replaced (rolling updates, retries) are left out and counted
    in `previous_tasks`.
    From Swarm's own task records (a manager's agent 1.8.0 or later;
    `tasks_known: false` and nulls otherwise):
    - `failures_1h` / `failures_24h`: tasks that failed or were rejected.
      Swarm retries a failed task as a new one, so this, not any container's
      restart count, is how a crash loop shows. Swarm keeps about 5 old tasks
      per replica, which bounds the count.
    - `crash_looping`: failures_1h reached the crash-loop count set under
      Settings → Thresholds (default 3).
    - `last_error`: the newest task that failed, was rejected, or carries an
      error while Swarm retries it — `error` is Swarm's own text (e.g. "No
      such image", "task: non-zero exit (1)"), with `exit_code`, `node`, `at`.
    - `trouble_since` / `trouble_age_s`: when the current trouble began — a
      crash loop's first failure in the hour, a short service's first failed
      task in its failing slots, a recovered service's first failure in 24h.
      Swarm keeps few old tasks, so a long outage reads as a lower bound.
    Services with a problem come first. `nodes` lists every swarm node once,
    with its role (manager/worker), availability and status.
    limit: default 100, max 500.
    """
    limit = clamp_limit(limit, 100, 500)
    now = now_utc()
    async with db_session() as db:
        services, nodes, host_names = await _swarm(db, now)
    seen, node_rows = set(), []
    for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
        if n.node_id in seen:
            continue
        seen.add(n.node_id)
        node_rows.append({"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
                          "leader": n.leader, "availability": n.availability, "status": n.status,
                          "reported_by": host_names.get(n.host_id)})
    node_rows.sort(key=lambda n: n["hostname"].lower())
    order = {"down": 0, "degraded": 1, "healthy": 2}
    def _rank(s):
        return (not s["crash_looping"], order.get(s["state"], 3), not s["failures_24h"], s["name"])
    def _error_row(err):
        if err is None:
            return None
        at = task_time(err)
        return {"error": err["error"], "message": err["message"], "state": err["state"],
                "exit_code": err["exit_code"], "node": err["node"],
                "at": iso(at), "age_s": age_s(at, now) if at else None}
    rows = [
        {
            "service_name": s["name"], "mode": s["mode"], "state": s["state"],
            "running": s["running"], "desired": s["desired"],
            "crash_looping": s["crash_looping"],
            "failures_1h": s["failures_1h"], "failures_24h": s["failures_24h"],
            "last_error": _error_row(s["last_error"]),
            "trouble_since": iso(s["trouble_since"]),
            "trouble_age_s": age_s(s["trouble_since"], now),
            "previous_tasks": s["previous_tasks"], "tasks_known": s["tasks_known"],
            "replicas": [
                {"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
                {"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
                 "status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
                 "cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
                for r in s["replicas"]
            ],
        }
        for s in sorted(services, key=_rank)
    ]
    return {"nodes": node_rows, **bounded(rows, limit, key="services")}
 async def search_container_logs(container: str = "", host: str = "", text: str = "",
                                range: str = "1h", stream: str = "", before: str = "",
                                context: int = 0, limit: int = 50) -> dict:
    """Search the container log lines Steward has retained, newest first.
    container: one container's name (as list_containers shows); empty searches
      every container (narrow with `host`). A container Docker has removed
      still works while its lines are retained: the reply then has
      `container_removed: true` and a `note` saying when it was last seen.
    host: a host's name or id; empty for all hosts.
    text: case-insensitive substring to match; empty matches every line.
    range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are
      kept per container up to a size and age limit, so older lines may be gone.
    stream: "stdout" or "stderr"; empty for both.
    before: an ISO time; returns only lines older than it. Pass a response's
      `next_before` to page further back.
    context: lines of surrounding log (0-5) to include with each match, from
      the same container. Only with `container` set.
    limit: matches to return, default 50, max 200.
    Lines longer than 2000 characters keep both ends, and the middle is
    replaced by a marker saying how much was cut.
    """
    since, key = lookback(range)
    limit = clamp_limit(limit, 50, 200)
    context = max(0, min(int(context or 0), 5))
    if context and not container:
        raise ValueError("context needs `container`: it shows lines around a match within one container's log.")
    if stream and stream not in ("stdout", "stderr"):
        raise ValueError(f"stream {stream!r} is not 'stdout' or 'stderr'.")
    before_dt = None
    if before:
        try:
            before_dt = datetime.fromisoformat(before.replace("Z", "+00:00"))
        except ValueError as exc:
            raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc
    async with db_session() as db:
        host_id = name = removed = None
        if container:
            c = await _pick_container(db, container, host, removed=True)
            host_id, name = c.host_id, c.name
            removed = c if isinstance(c, RemovedContainer) else None
        elif host:
            hosts = list((await db.execute(select(Host))).scalars())
            host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
        rows = await docker_svc.search_logs(db, since=since, host_id=host_id, name=name, text=text,
                                            stream=stream, before=before_dt, limit=limit + 1)
        more = len(rows) > limit
        rows = rows[:limit]
        host_names = await _host_names(db)
        matches = []
        for r in rows:
            m = {"at": iso(r.ts), "container_name": r.container_name, "host_id": r.host_id,
                 "host_name": host_names.get(r.host_id), "stream": r.stream, "line": cut_middle(r.line)}
            if context:
                pre, post = await docker_svc.log_context(db, r.host_id, r.container_name, r.ts, context)
                m["before"] = [cut_middle(x.line) for x in pre]
                m["after"] = [cut_middle(x.line) for x in post]
            matches.append(m)
    out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more}
    if removed is not None:
        out["container_removed"] = True
        out["note"] = _removed_note(removed)
        if removed.last_seen < since:
            out["note"] += " That is before this range; pass a longer `range`."
    if more:
        # Full precision: `at` is cut to the second for reading, and a cursor
        # cut the same way would skip lines logged earlier in that second.
        out["next_before"] = rows[-1].ts.isoformat()
        out["more"] = "More matches are older; pass next_before as `before`, or narrow with `text`."
    return out
 async def get_docker_disk_usage(host: str = "") -> dict:
    """Docker disk usage per host: images, containers, volumes and build cache, in bytes.
    host: a host's name or id; empty for every host. `images_reclaimable_bytes`
    is what pruning unused images would free. An admin can prune from
    Steward's Docker → Disk page.
    """
    async with db_session() as db:
        summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
        host_names = await _host_names(db)
        if host:
            hosts = list((await db.execute(select(Host))).scalars())
            hid = pick(host, hosts, kind="host", list_tool="list_hosts").id
            summaries = [s for s in summaries if s.host_id == hid]
    rows = [
        {
            "host_id": s.host_id, "host_name": host_names.get(s.host_id),
            "images_total": s.images_total, "images_active": s.images_active,
            "images_bytes": s.images_size, "images_reclaimable_bytes": s.images_reclaimable,
            "containers_count": s.containers_count, "containers_bytes": s.containers_size,
            "volumes_count": s.volumes_count, "volumes_bytes": s.volumes_size,
            "build_cache_bytes": s.build_cache_size,
            "reported_at": iso(s.updated_at), "reported_age_s": age_s(s.updated_at),
        }
        for s in summaries
    ]
    rows.sort(key=lambda r: -(r["images_reclaimable_bytes"] or 0))
    return {"hosts": rows, "total": len(rows)}
 PRUNE_BUILTIN_PLAYBOOK = "maintenance/docker_prune.yml"
 _STOPPED = ("exited", "created", "dead")
 def prune_preview(what: str, usage, stopped: int) -> dict:
    """What a prune of `what` would remove, from the host's latest disk report."""
    out: dict = {}
    if what in ("containers", "system"):
        out["stopped_containers"] = stopped
    if what == "images":
        out["unused_images_bytes"] = usage.images_reclaimable if usage else None
    if what == "system":
        out["dangling_images"] = "yes (unused but tagged images stay)"
        out["build_cache_bytes"] = usage.build_cache_size if usage else None
        out["unused_networks"] = "yes"
    return out
 async def prune_docker(host: str, what: str, check: bool = True) -> dict:
    """Reclaim Docker disk on a host. Previews unless check=false.
    host: the Steward host's name or id. It needs a linked Ansible target
      (list_inventory shows the links); the prune runs through Ansible as
      Steward's built-in docker_prune playbook.
    what: "containers" removes stopped containers; "images" removes ALL
      unused images, not only dangling ones; "system" is `docker system
      prune -f`: stopped containers, dangling images, unused networks and
      build cache. Volumes are never pruned.
    check: true (default) starts nothing and returns `would_remove`, sized
      from the host's latest Docker disk report. Ansible's own check mode
      skips the prune commands, so a check-mode run would show nothing.
      Pass false to run the prune.
    Admin only, as on the Docker → Disk page. A real prune returns `run_id`
    (follow it with wait_for_run) and is recorded in the audit log.
    """
    what = (what or "").strip()
    if what not in docker_svc.PRUNE_TARGETS:
        raise ValueError(f"what {what!r} is not one of {', '.join(docker_svc.PRUNE_TARGETS)}.")
    async with db_session() as db:
        h = pick(host, (await db.execute(select(Host))).scalars(), kind="host", list_tool="list_hosts")
        target = (await db.execute(
            select(AnsibleTarget).where(AnsibleTarget.host_id == h.id))).scalars().first()
        usage = await db.get(DockerDiskUsage, h.id)
        stopped = sum(1 for c in await docker_svc.list_containers(db, h.id)
                      if (c.status or "").lower() in _STOPPED)
    if target is None:
        raise ValueError(f"host {h.name!r} has no linked Ansible target, so Steward cannot reach it "
                         "to prune. Link one under Ansible → Inventory. Nothing was run.")
    preview = {
        "host": h.name,
        "what": what,
        "ansible_target": target.name,
        "would_remove": prune_preview(what, usage, stopped),
        "disk_report_age_s": age_s(usage.updated_at) if usage else None,
    }
    if check:
        return {**preview, "check": True,
                "summary": f"Nothing was run. Pass check=false to prune {what} on {h.name}."}
    caller = current_caller()
    run, _src, err = await trigger_run(
        mcp_app(), source_name=BUILTIN_SOURCE_NAME, playbook_path=PRUNE_BUILTIN_PLAYBOOK,
        inventory_scope=f"steward:target:{target.id}",
        params={"extra_vars_map": docker_svc.prune_extra_vars(what)},
        triggered_by=caller.user_id)
    if err:
        raise ValueError(f"the prune could not start: {err}.")
    await audit_mcp("docker.prune.started", "host", h.name,
                    {"host_id": h.id, "what": what, "run_id": run.id})
    return {**preview, "check": False, "run_id": run.id,
            "next": f'wait_for_run(run_id="{run.id}")'}
 def register_tools(register) -> None:
    for fn in (list_containers, get_container, list_swarm_services,
               search_container_logs, get_docker_disk_usage):
        register(fn, role=UserRole.viewer, read_only=True)
    register(prune_docker, role=UserRole.admin, read_only=False)
@@ -1,48 +0,0 @@
 """Docker container logs table
 Adds docker_logs — one row per container log line, pushed by the host agent and
 folded into its metrics push. Time-series, host-scoped (container names are only
 unique within a host). Bounded by a per-container size+age ring in retention, so
 a chatty container keeps a shorter window rather than growing without limit.
 Additive create_table + twin indexes (viewer lookup by (host, container, ts);
 age-cutoff prune by ts).
 Revision ID: docker_009_container_logs
 Revises: docker_008_bigint_mem
 Create Date: 2026-07-19
 """
 from typing import Sequence, Union
 from alembic import op
 import sqlalchemy as sa
 revision: str = "docker_009_container_logs"
 down_revision: Union[str, None] = "docker_008_bigint_mem"
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 def upgrade() -> None:
    op.create_table(
        "docker_logs",
        sa.Column("id", sa.String(length=36), nullable=False),
        sa.Column("host_id", sa.String(length=36), nullable=False),
        sa.Column("container_name", sa.String(length=255), nullable=False),
        sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
        sa.Column("stream", sa.String(length=8), nullable=False, server_default="stdout"),
        sa.Column("line", sa.Text(), nullable=False, server_default=""),
        sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
        sa.PrimaryKeyConstraint("id"),
    )
    op.create_index("ix_docker_logs_host_id", "docker_logs", ["host_id"])
    op.create_index("ix_docker_logs_container_name", "docker_logs", ["container_name"])
    op.create_index("ix_docker_logs_host_container_time",
                    "docker_logs", ["host_id", "container_name", "ts"])
    op.create_index("ix_docker_logs_ts", "docker_logs", ["ts"])
 def downgrade() -> None:
    op.drop_index("ix_docker_logs_ts", table_name="docker_logs")
    op.drop_index("ix_docker_logs_host_container_time", table_name="docker_logs")
    op.drop_index("ix_docker_logs_container_name", table_name="docker_logs")
    op.drop_index("ix_docker_logs_host_id", table_name="docker_logs")
    op.drop_table("docker_logs")
@@ -1,30 +0,0 @@
 """Docker swarm service tasks column
 Adds docker_swarm_services.tasks_json — each service's newest tasks as a
 manager's agent (≥ 1.8.0) reports them: slot, node, desired and current state,
 Swarm's error and message, exit code and time. Lets Steward tell a task Swarm
 replaced from one that crashed, count a service's failures across task
 retries, and say why its last task failed. Additive column; no DROP+recreate.
 Revision ID: docker_010_swarm_tasks
 Revises: docker_009_container_logs
 Create Date: 2026-10-06
 """
 from typing import Sequence, Union
 from alembic import op
 import sqlalchemy as sa
 revision: str = "docker_010_swarm_tasks"
 down_revision: Union[str, None] = "docker_009_container_logs"
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 def upgrade() -> None:
    op.add_column("docker_swarm_services",
                  sa.Column("tasks_json", sa.Text, nullable=False,
                            server_default="[]"))
 def downgrade() -> None:
    op.drop_column("docker_swarm_services", "tasks_json")
@@ -163,43 +163,6 @@ class DockerEvent(Base):
    )
 class DockerLog(Base):
    """Container log lines pushed by the host agent — one row per line.
    Time-series, scoped to the reporting host (container names are only unique
    within a host, same identity as docker_metrics). The agent tails each running
    container incrementally and folds new lines into its metrics push; `ts` is the
    line's own Docker timestamp. Bounded by a per-container size+age ring
    (retention), so a chatty container just keeps a shorter window rather than
    growing without limit.
    """
    __tablename__ = "docker_logs"
    id: Mapped[str] = mapped_column(
        String(36), primary_key=True, default=lambda: str(uuid.uuid4())
    )
    host_id: Mapped[str] = mapped_column(
        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
    )
    container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
    ts: Mapped[datetime] = mapped_column(
        DateTime(timezone=True), nullable=False,
        default=lambda: datetime.now(timezone.utc),
    )
    stream: Mapped[str] = mapped_column(String(8), nullable=False, default="stdout")
    # stdout | stderr
    line: Mapped[str] = mapped_column(Text, nullable=False, default="")
    # Viewer filters on (host_id, container_name) and sorts by time; the ring
    # prune walks the same key newest-first. A second index on `ts` alone serves
    # the age-cutoff delete. Twin-index idiom, mirroring docker_events.
    __table_args__ = (
        Index("ix_docker_logs_host_container_time",
              "host_id", "container_name", "ts"),
        Index("ix_docker_logs_ts", "ts"),
    )
 class DockerSwarmService(Base):
    """A Swarm service as seen by a manager host: desired vs running replicas.
@@ -223,11 +186,6 @@ class DockerSwarmService(Base):
    # A manager sees every task, so this captures cross-node placement that the
    # local docker_containers rows (this host only) can't.
    placement_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
    # The service's newest tasks, newest first, as the manager's agent shaped
    # them: [{"task_id", "slot", "node_id", "desired_state", "state", "message",
    # "error", "exit_code", "at"}], JSON. Swarm's own record of which tasks it
    # replaced and why each one stopped; "[]" from agents older than 1.8.0.
    tasks_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
    updated_at: Mapped[datetime] = mapped_column(
        DateTime(timezone=True), nullable=False,
        default=lambda: datetime.now(timezone.utc),
@@ -38,8 +38,6 @@ async def run_docker_retention(
    events_days: int,
    metrics_raw_days: int,
    metrics_rollup_days: int,
    logs_retention_days: int = 3,
    logs_max_bytes_per_container: int = 5_000_000,
    now: datetime | None = None,
 ) -> dict:
    """Roll up + prune Docker time-series. Returns a counts dict for logging.
@@ -49,22 +47,18 @@ async def run_docker_retention(
       re-run is idempotent, then delete those raw rows.
    2. Prune rolled-up rows older than the rollup window.
    3. Prune docker_events older than the events window.
    4. Prune docker_logs with a per-container size+age ring (m79): drop lines
       older than the age window, then keep only the newest ~cap bytes per
       (host, container).
    """
    from datetime import timezone
    from sqlalchemy import delete, func, select
    from sqlalchemy.dialects.postgresql import insert as pg_insert
-    from .models import DockerEvent, DockerLog, DockerMetric, DockerMetricHourly
+    from .models import DockerEvent, DockerMetric, DockerMetricHourly
    if now is None:
        now = datetime.now(timezone.utc)
    rolled = rolled_rows = events_pruned = rollup_pruned = 0
    logs_age_pruned = logs_size_pruned = 0
    # ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
    raw_cutoff = _rollup_cutoff(now, metrics_raw_days)
@@ -126,35 +120,9 @@ async def run_docker_retention(
    )
    events_pruned = res.rowcount or 0
    # ── 4. Container-log ring: age cutoff, then per-container byte cap (m79) ──
    logs_cutoff = now - timedelta(days=logs_retention_days)
    res = await session.execute(
        delete(DockerLog).where(DockerLog.ts < logs_cutoff)
    )
    logs_age_pruned = res.rowcount or 0
    # Size ring: per (host, container), sum line bytes newest-first; delete a row
    # once its strictly-newer siblings already fill the cap. Using the EXCLUSIVE
    # prefix (running total minus this row) means the newest row always survives,
    # so a single line larger than the cap is never wiped out.
    running = func.sum(func.length(DockerLog.line)).over(
        partition_by=[DockerLog.host_id, DockerLog.container_name],
        order_by=[DockerLog.ts.desc(), DockerLog.id.desc()],
    )
    prefix_excl = (running - func.length(DockerLog.line)).label("prefix_excl")
    ranked = select(DockerLog.id, prefix_excl).subquery()
    over_cap = select(ranked.c.id).where(
        ranked.c.prefix_excl >= logs_max_bytes_per_container)
    res = await session.execute(
        delete(DockerLog).where(DockerLog.id.in_(over_cap))
    )
    logs_size_pruned = res.rowcount or 0
    return {
        "buckets_rolled": rolled,
        "raw_rows_rolled": rolled_rows,
        "rollup_pruned": rollup_pruned,
        "events_pruned": events_pruned,
        "logs_age_pruned": logs_age_pruned,
        "logs_size_pruned": logs_size_pruned,
    }
@@ -3,39 +3,54 @@ from __future__ import annotations
 import json
 from datetime import datetime, timedelta, timezone
-from quart import (
+from quart import Blueprint, current_app, render_template, request
    Blueprint, current_app, jsonify, redirect, render_template, request,
    session, url_for,
 )
 from sqlalchemy import func, select
 from steward.auth.middleware import require_role
 from steward.models.users import UserRole
 from steward.models.hosts import Host
 from steward.core.formatting import human_bytes, human_duration, seconds_since
 from steward.core.time_range import parse_range, DEFAULT_RANGE
 from . import service as docker_svc
 from .dedup import dedup_by_container_id
-from .swarm_view import build_swarm_services, parse_tasks, task_summary
+from .swarm_view import build_swarm_services
 from .models import (
-    DockerContainer, DockerDiskUsage, DockerEvent, DockerImage,
+    DockerContainer, DockerDiskUsage, DockerEvent, DockerImage, DockerMetric,
    DockerSwarmNode, DockerSwarmService,
 )
 docker_bp = Blueprint("docker", __name__, template_folder="templates")
-def _error(status: int, code: str, detail: str | None = None):
+def _human_bytes(n: int | None) -> str:
-    body: dict = {"ok": False, "error": code}
+    """Compact binary size string (e.g. '1.4 GiB', '512 MiB', '0 B')."""
-    if detail:
+    if n is None:
-        body["detail"] = detail
+        return "—"
-    return jsonify(body), status
+    size = float(n)
    for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
        if abs(size) < 1024.0 or unit == "TiB":
            if unit == "B":
                return f"{int(size)} {unit}"
            return f"{size:.1f} {unit}"
        size /= 1024.0
    return f"{size:.1f} TiB"
-def _uptime(started_at: datetime | None) -> str | None:
+def _human_uptime(started_at: datetime | None) -> str | None:
-    """How long a container has run ('3d 4h'), or None if unknown."""
+    """Compact 'how long running' string (e.g. '3d 4h', '5h 12m', '8m')."""
-    secs = seconds_since(started_at)
+    if started_at is None:
-    return None if secs is None or secs < 0 else human_duration(secs)
+        return None
    if started_at.tzinfo is None:
        started_at = started_at.replace(tzinfo=timezone.utc)
    secs = int((datetime.now(timezone.utc) - started_at).total_seconds())
    if secs < 0:
        return None
    d, rem = divmod(secs, 86400)
    h, rem = divmod(rem, 3600)
    m, _ = divmod(rem, 60)
    if d:
        return f"{d}d {h}h"
    if h:
        return f"{h}h {m}m"
    return f"{m}m"
 def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
@@ -59,6 +74,38 @@ def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
    )
 def _bucket(values: list[float], target: int = 40) -> list[float]:
    """Bucket-average a series down to ~target points (DB-agnostic, in Python).
    Replaces the old SQL strftime() bucketing, which was SQLite-only and broke
    on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of
    rows — averaging into a readable point count keeps the sparkline's shape.
    """
    n = len(values)
    if n <= target:
        return values
    size = (n + target - 1) // target
    out: list[float] = []
    for i in range(0, n, size):
        chunk = values[i:i + size]
        out.append(sum(chunk) / len(chunk))
    return out
 async def _container_history(db, host_id: str, name: str, since) -> tuple[list, list]:
    """Return (cpu_series, mem_series) sparkline-ready for one host's container."""
    rows = (await db.execute(
        select(DockerMetric.cpu_pct, DockerMetric.mem_pct)
        .where(DockerMetric.host_id == host_id)
        .where(DockerMetric.container_name == name)
        .where(DockerMetric.scraped_at >= since)
        .order_by(DockerMetric.scraped_at)
    )).all()
    cpu = _bucket([r.cpu_pct or 0.0 for r in rows])
    mem = _bucket([r.mem_pct or 0.0 for r in rows])
    return cpu, mem
@docker_bp.get("/")
@require_role(UserRole.viewer)
 async def index():
@@ -80,6 +127,15 @@ async def index():
    )
 async def _host_map(db, host_ids: set[str]) -> dict[str, Host]:
    if not host_ids:
        return {}
    return {
        h.id: h for h in (await db.execute(
            select(Host).where(Host.id.in_(host_ids)))).scalars().all()
    }
@docker_bp.get("/rows")
@require_role(UserRole.viewer)
 async def rows():
@@ -88,10 +144,15 @@ async def rows():
    since, range_key = parse_range(request.args.get("range"))
    async with current_app.db_sessionmaker() as db:
-        containers = await docker_svc.list_containers(db)
+        containers = list((await db.execute(
            select(DockerContainer).order_by(
                (DockerContainer.status == "running").desc(),
                DockerContainer.name,
            )
        )).scalars())
        services = list((await db.execute(select(DockerSwarmService))).scalars())
        nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
-        hosts = await docker_svc.host_map(
+        hosts = await _host_map(
            db,
            {c.host_id for c in containers}
            | {n.host_id for n in nodes} | {s.host_id for s in services},
@@ -102,9 +163,7 @@ async def rows():
        # from the managers' placement). Non-swarm containers keep the host view.
        node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
        host_name = {h.id: h.name for h in hosts.values()}
-        swarm_view = build_swarm_services(
+        swarm_view = build_swarm_services(containers, services, node_hostname, host_name)
            containers, services, node_hostname, host_name,
            crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
        standalone = [c for c in containers if not c.service_name]
        groups: dict[str, dict] = {}
@@ -118,11 +177,11 @@ async def rows():
                    "host_name": host.name if host else c.host_id,
                    "containers": [], "running": 0, "stopped": 0,
                }
-            cpu_hist, mem_hist = await docker_svc.container_history(db, c.host_id, c.name, since)
+            cpu_hist, mem_hist = await _container_history(db, c.host_id, c.name, since)
            g["containers"].append({
                "container": c,
                "ports": json.loads(c.ports_json) if c.ports_json else [],
-                "uptime": _uptime(c.started_at) if c.status == "running" else None,
+                "uptime": _human_uptime(c.started_at) if c.status == "running" else None,
                "sparkline_cpu": _sparkline(cpu_hist),
                "sparkline_mem": _sparkline(mem_hist),
            })
@@ -180,14 +239,14 @@ async def widget():
        )).scalars()))
        services = list((await db.execute(select(DockerSwarmService))).scalars())
        nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
-        # Recent failures are more actionable than a static "stopped" tally. Read
+        # Recent failures are more actionable than a static "stopped" tally. Count
        # DISTINCT container names so the two managers' duplicate die/oom events
-        # don't double it; swarm tasks are counted on their service below.
+        # don't double it.
-        dead_names = list((await db.execute(
+        failed_24h = (await db.execute(
-            select(func.distinct(DockerEvent.container_name))
+            select(func.count(func.distinct(DockerEvent.container_name)))
            .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since_24h)
-        )).scalars())
+        )).scalar() or 0
-        hosts = await docker_svc.host_map(
+        hosts = await _host_map(
            db,
            {c.host_id for c in all_containers}
            | {n.host_id for n in nodes} | {s.host_id for s in services},
@@ -197,10 +256,7 @@ async def widget():
    # host); non-swarm containers stay grouped by host.
    node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
    host_name = {h.id: h.name for h in hosts.values()}
-    swarm_services = build_swarm_services(
+    swarm_services = build_swarm_services(all_containers, services, node_hostname, host_name)["services"]
        all_containers, services, node_hostname, host_name,
        crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"]
    failed_24h = docker_svc.failed_24h(dead_names, swarm_services)
    standalone = [c for c in all_containers if not c.service_name]
    running = [c for c in standalone if c.status == "running"]
@@ -245,7 +301,7 @@ async def widget_resources():
            .where(DockerContainer.status == "running")
            .order_by(DockerContainer.cpu_pct.desc().nullslast())
        )).scalars()))[:limit]
-        hosts = await docker_svc.host_map(db, {c.host_id for c in containers})
+        hosts = await _host_map(db, {c.host_id for c in containers})
    rows_data = [
        {"c": c, "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id}
@@ -268,7 +324,14 @@ async def host_panel(host_id: str):
    don't carry an empty card.
    """
    async with current_app.db_sessionmaker() as db:
-        containers = await docker_svc.list_containers(db, host_id)
+        containers = list((await db.execute(
            select(DockerContainer)
            .where(DockerContainer.host_id == host_id)
            .order_by(
                (DockerContainer.status == "running").desc(),
                DockerContainer.name,
            )
        )).scalars())
    if not containers:
        return ""
@@ -298,7 +361,15 @@ async def container_detail(host_id: str, name: str):
    async with current_app.db_sessionmaker() as db:
        c = await db.get(DockerContainer, (host_id, name))
        host = await db.get(Host, host_id)
-        events = await docker_svc.container_events(db, host_id, name) if c is not None else []
+        events = []
        if c is not None:
            events = list((await db.execute(
                select(DockerEvent)
                .where(DockerEvent.host_id == host_id)
                .where(DockerEvent.container_name == name)
                .order_by(DockerEvent.at.desc())
                .limit(50)
            )).scalars())
    if c is None:
        return await render_template(
@@ -313,9 +384,9 @@ async def container_detail(host_id: str, name: str):
        host_id=host_id,
        name=name,
        ports=json.loads(c.ports_json) if c.ports_json else [],
-        uptime=_uptime(c.started_at) if c.status == "running" else None,
+        uptime=_human_uptime(c.started_at) if c.status == "running" else None,
-        net_rx=human_bytes(c.net_rx_bytes), net_tx=human_bytes(c.net_tx_bytes),
+        net_rx=_human_bytes(c.net_rx_bytes), net_tx=_human_bytes(c.net_tx_bytes),
-        blk_read=human_bytes(c.blk_read_bytes), blk_write=human_bytes(c.blk_write_bytes),
+        blk_read=_human_bytes(c.blk_read_bytes), blk_write=_human_bytes(c.blk_write_bytes),
        events=[
            {"event": e.event, "detail": e.detail, "at": e.at,
             "glyph": _EVENT_STYLE.get(e.event, ("•", "var(--text-muted)"))[0],
@@ -346,7 +417,7 @@ async def swarm():
            select(DockerSwarmNode).order_by(
                DockerSwarmNode.role, DockerSwarmNode.hostname)
        )).scalars())
-        hosts = await docker_svc.host_map(db, {s.host_id for s in services} | {n.host_id for n in nodes})
+        hosts = await _host_map(db, {s.host_id for s in services} | {n.host_id for n in nodes})
    # ── Group reporting managers into swarms by shared node membership ──────────
    manager_ids = {s.host_id for s in services} | {n.host_id for n in nodes}
@@ -392,8 +463,6 @@ async def swarm():
                node_by_id[n.node_id] = n
        node_names = {nid: (nrow.hostname or nid) for nid, nrow in node_by_id.items()}
        now = datetime.now(timezone.utc)
        deaths = docker_svc.crash_loop_deaths(current_app.config)
        svc_dicts = []
        for s in sorted(svc_by_name.values(), key=lambda s: s.service_name):
            placement = []
@@ -409,10 +478,7 @@ async def swarm():
                "healthy": s.running >= s.desired and s.desired > 0,
                "degraded": 0 < s.running < s.desired,
                "down": s.running == 0 and s.desired > 0,
                "state": ("healthy" if s.running >= s.desired and s.desired > 0
                          else "degraded" if s.running > 0 else "down"),
                "placement": placement,
                **task_summary(parse_tasks(s), now, deaths, node_names),
            })
        node_rows = sorted(
            node_by_id.values(), key=lambda n: (n.role, (n.hostname or n.node_id).lower())
@@ -433,13 +499,9 @@ async def swarm():
 async def disk():
    """Image/disk usage page: reclaimable space, per-image sizes, stopped count.
-    Admins can reclaim space per host via the prune buttons, which fire the
+    Read-only — prune actions are deferred to the cleanup-actions milestone, so
-    audited bundled prune playbook through Ansible (m78) — the collection agent
+    this surfaces the numbers and notes where reclaim lives.
    itself stays read-only.
    """
    from steward.core.capabilities import has_capability
    from steward.models.ansible_inventory import AnsibleTarget
    async with current_app.db_sessionmaker() as db:
        summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
        images = list((await db.execute(
@@ -453,23 +515,13 @@ async def disk():
        stopped_by_host: dict[str, int] = {}
        for hid in stopped_rows:
            stopped_by_host[hid] = stopped_by_host.get(hid, 0) + 1
-        hosts = await docker_svc.host_map(db, {s.host_id for s in summaries})
+        hosts = await _host_map(db, {s.host_id for s in summaries})
        # Which hosts have a linked Ansible target — gates the prune buttons
        # (a target is required to route the playbook run to that host).
        host_ids = {s.host_id for s in summaries}
        target_by_host: dict[str, str] = {}
        if host_ids:
            for hid, tid in (await db.execute(
                select(AnsibleTarget.host_id, AnsibleTarget.id)
                .where(AnsibleTarget.host_id.in_(host_ids))
            )).all():
                target_by_host[hid] = tid
    images_by_host: dict[str, list] = {}
    for im in images:
        images_by_host.setdefault(im.host_id, []).append({
-            "repo_tag": im.repo_tag, "size": human_bytes(im.size),
+            "repo_tag": im.repo_tag, "size": _human_bytes(im.size),
-            "shared": human_bytes(im.shared_size), "containers": im.containers,
+            "shared": _human_bytes(im.shared_size), "containers": im.containers,
            "reclaimable": im.containers == 0,
        })
@@ -479,71 +531,22 @@ async def disk():
            "host_name": hosts[s.host_id].name if s.host_id in hosts else s.host_id,
            "summary": {
                "images_total": s.images_total, "images_active": s.images_active,
-                "images_size": human_bytes(s.images_size),
+                "images_size": _human_bytes(s.images_size),
-                "images_reclaimable": human_bytes(s.images_reclaimable),
+                "images_reclaimable": _human_bytes(s.images_reclaimable),
-                "layers_size": human_bytes(s.layers_size),
+                "layers_size": _human_bytes(s.layers_size),
                "containers_count": s.containers_count,
-                "containers_size": human_bytes(s.containers_size),
+                "containers_size": _human_bytes(s.containers_size),
                "volumes_count": s.volumes_count,
-                "volumes_size": human_bytes(s.volumes_size),
+                "volumes_size": _human_bytes(s.volumes_size),
-                "build_cache_size": human_bytes(s.build_cache_size),
+                "build_cache_size": _human_bytes(s.build_cache_size),
            },
            "stopped": stopped_by_host.get(s.host_id, 0),
            "images": images_by_host.get(s.host_id, []),
            "has_target": s.host_id in target_by_host,
        }
        for s in summaries
    ]
    host_groups.sort(key=lambda g: g["host_name"].lower())
-    return await render_template(
+    return await render_template("docker/disk.html", host_groups=host_groups)
        "docker/disk.html", host_groups=host_groups,
        ansible_available=has_capability("ansible.run_playbook"),
    )
@docker_bp.post("/disk/<host_id>/prune")
@require_role(UserRole.admin)
 async def disk_prune(host_id: str):
    """Reclaim Docker disk on a host by firing the bundled prune playbook via
    Ansible (audited, admin-gated) — the collection agent stays read-only (m78).
    `target` selects the scope: containers | images | system.
    """
    from steward.core.capabilities import has_capability, invoke_capability
    from steward.ansible.sources import BUILTIN_SOURCE_NAME
    from steward.models.ansible_inventory import AnsibleTarget
    if not has_capability("ansible.run_playbook"):
        return _error(400, "ansible_unavailable", "Ansible is not available")
    form = await request.form
    prune_target = (form.get("target", "") or "").strip()
    if prune_target not in docker_svc.PRUNE_TARGETS:
        return _error(400, "bad_target", "Unknown prune target")
    async with current_app.db_sessionmaker() as db:
        target = (await db.execute(
            select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
        )).scalar_one_or_none()
    if target is None:
        return _error(400, "no_target",
                      "Link an Ansible target to this host before pruning")
    # extra_vars_map outranks the playbook's own `vars:` defaults.
    extra_vars = docker_svc.prune_extra_vars(prune_target)
    actor_role = UserRole(session.get("user_role", "viewer"))
    run, _source, err = await invoke_capability(
        "ansible.run_playbook", actor_role,
        current_app._get_current_object(),  # type: ignore[attr-defined]
        source_name=BUILTIN_SOURCE_NAME,
        playbook_path="maintenance/docker_prune.yml",
        inventory_scope=f"steward:target:{target.id}",
        params={"extra_vars_map": extra_vars},
        triggered_by=session.get("user_id"),
    )
    if err:
        return _error(400, "prune_failed", err)
    return redirect(url_for("ansible.run_detail", run_id=run.id))
@docker_bp.get("/container/<host_id>/<name>/history")
@@ -552,7 +555,7 @@ async def container_history(host_id: str, name: str):
    """HTMX fragment: CPU + memory sparklines for the selected time range."""
    since, range_key = parse_range(request.args.get("range"))
    async with current_app.db_sessionmaker() as db:
-        cpu_hist, mem_hist = await docker_svc.container_history(db, host_id, name, since)
+        cpu_hist, mem_hist = await _container_history(db, host_id, name, since)
    return await render_template(
        "docker/_container_history.html",
        sparkline_cpu=_sparkline(cpu_hist, width=320, height=48),
@@ -560,36 +563,3 @@ async def container_history(host_id: str, name: str):
        have_data=len(cpu_hist) >= 2,
        range_key=range_key,
    )
 # Newest lines returned per fetch — the retained window is bounded by the ring,
 # but a container can still hold thousands of lines; cap what one fetch renders.
 _LOG_TAIL_DEFAULT = 500
@docker_bp.get("/container/<host_id>/<name>/logs")
@require_role(UserRole.viewer)
 async def container_logs(host_id: str, name: str):
    """Full log-viewer page for one container (lines load via an HTMX fragment)."""
    async with current_app.db_sessionmaker() as db:
        host = await db.get(Host, host_id)
    return await render_template(
        "docker/container_logs.html", host=host, host_id=host_id, name=name,
    )
@docker_bp.get("/container/<host_id>/<name>/logs/lines")
@require_role(UserRole.viewer)
 async def container_logs_lines(host_id: str, name: str):
    """HTMX fragment: the recent retained log lines, stream + text filtered.
    Polled every few seconds by the viewer for near-live follow."""
    stream = request.args.get("stream", "all")
    query = (request.args.get("q") or "").strip()
    async with current_app.db_sessionmaker() as db:
        rows = await docker_svc.query_logs(db, host_id, name, stream, query, _LOG_TAIL_DEFAULT)
    return await render_template(
        "docker/_container_logs_lines.html",
        lines=[{"ts": r.ts, "stream": r.stream, "line": r.line} for r in rows],
        name=name, tail=_LOG_TAIL_DEFAULT,
        filtered=bool(query) or stream in ("stdout", "stderr"),
    )
@@ -1,280 +0,0 @@
 """Docker queries shared by the plugin's pages and its MCP tools.
 Each function takes an open session and returns rows or plain data, with no
 Quart request context, so the routes and the /mcp tools (which run outside
 Quart's pipeline) call the same code.
 """
 from __future__ import annotations
 from datetime import datetime
 from sqlalchemy import func, select
 from steward.core.time_range import bucket_seconds
 from steward.models.hosts import Host
 from .models import DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly
 CRASH_LOOP_DEATHS = 3  # default; Settings → Thresholds sets it
 def crash_loop_deaths(config) -> int:
    """Deaths within an hour that make a container or a swarm service a crash
    loop, from Settings → Thresholds (app.config["THRESHOLDS"])."""
    th = (config.get("THRESHOLDS") or {}).get("crash_loop") or {}
    return int(th.get("warn") or CRASH_LOOP_DEATHS)
 def failed_24h(dead_names, swarm_services) -> int:
    """Failures in the last 24h for the dashboard: distinct plain containers
    that died or were OOM-killed, plus each swarm service's failed tasks.
    A swarm task's container dies whenever Swarm replaces it (a rolling update
    kills it with 137), so its die event is not a failure. Services with
    Swarm's task records count their failed tasks instead; a service without
    them (an agent older than 1.8.0) falls back to its tasks' die events.
    """
    by_prefix = {s["name"] + ".": s for s in swarm_services}
    plain, fallback = 0, set()
    for name in dead_names:
        svc = next((s for p, s in by_prefix.items() if name.startswith(p)), None)
        if svc is None:
            plain += 1
        elif not svc["tasks_known"]:
            fallback.add(name)
    tasks = sum(s["failures_24h"] or 0 for s in swarm_services if s["tasks_known"])
    return plain + len(fallback) + tasks
 def bucket(values: list[float], target: int = 40) -> list[float]:
    """Bucket-average a series down to ~target points (DB-agnostic, in Python).
    Replaces the old SQL strftime() bucketing, which was SQLite-only and broke
    on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of
    rows — averaging into a readable point count keeps the sparkline's shape.
    """
    n = len(values)
    if n <= target:
        return values
    size = (n + target - 1) // target
    out: list[float] = []
    for i in range(0, n, size):
        chunk = values[i:i + size]
        out.append(sum(chunk) / len(chunk))
    return out
 async def container_history(db, host_id: str, name: str, since) -> tuple[list, list]:
    """Return (cpu_series, mem_series) sparkline-ready for one host's container."""
    rows = (await db.execute(
        select(DockerMetric.cpu_pct, DockerMetric.mem_pct)
        .where(DockerMetric.host_id == host_id)
        .where(DockerMetric.container_name == name)
        .where(DockerMetric.scraped_at >= since)
        .order_by(DockerMetric.scraped_at)
    )).all()
    cpu = bucket([r.cpu_pct or 0.0 for r in rows])
    mem = bucket([r.mem_pct or 0.0 for r in rows])
    return cpu, mem
 async def host_map(db, host_ids: set[str]) -> dict[str, Host]:
    if not host_ids:
        return {}
    return {
        h.id: h for h in (await db.execute(
            select(Host).where(Host.id.in_(host_ids)))).scalars().all()
    }
 def escape_like(text: str) -> str:
    """Make `text` match literally inside a LIKE/ILIKE pattern (escape char \\)."""
    return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
 async def list_containers(db, host_id: str | None = None) -> list[DockerContainer]:
    """Containers, running first then by name; one host's when host_id is given."""
    stmt = select(DockerContainer)
    if host_id is not None:
        stmt = stmt.where(DockerContainer.host_id == host_id)
    stmt = stmt.order_by((DockerContainer.status == "running").desc(), DockerContainer.name)
    return list((await db.execute(stmt)).scalars())
 async def container_events(db, host_id: str, name: str, limit: int = 50) -> list[DockerEvent]:
    """A container's lifecycle events (start/stop/die/oom/health), newest first."""
    return list((await db.execute(
        select(DockerEvent)
        .where(DockerEvent.host_id == host_id)
        .where(DockerEvent.container_name == name)
        .order_by(DockerEvent.at.desc())
        .limit(limit)
    )).scalars())
 async def query_logs(db, host_id: str, name: str, stream: str, query: str,
                     limit: int) -> list:
    """The recent retained lines for one container, newest-first, optionally
    filtered by stream and a case-insensitive substring.
    Newest-first is deliberate: the viewer replaces the list on each poll, which
    resets scroll to the top — so the latest lines stay visible without any
    scroll handling, and older lines are a scroll away.
    """
    stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line)
            .where(DockerLog.host_id == host_id)
            .where(DockerLog.container_name == name))
    if stream in ("stdout", "stderr"):
        stmt = stmt.where(DockerLog.stream == stream)
    if query:
        stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\"))
    stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
    return (await db.execute(stmt)).all()
 CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes")
 async def container_points(db, host_id: str, name: str, metric: str, since,
                           target: int = 120) -> list[tuple]:
    """(time, average) points of one container metric since `since`, oldest first.
    Raw samples are bucket-averaged in SQL; the part of the range older than
    the oldest raw sample (already rolled up and pruned by retention) comes
    from the hourly table.
    """
    if metric not in CONTAINER_SERIES:
        raise ValueError(f"metric must be one of {', '.join(CONTAINER_SERIES)}")
    raw_col = getattr(DockerMetric, metric)
    width_s = bucket_seconds(since, target)
    rbucket = func.date_bin(
        func.make_interval(0, 0, 0, 0, 0, 0, width_s), DockerMetric.scraped_at, func.to_timestamp(0),
    ).label("bucket")
    raw = (await db.execute(
        select(rbucket, func.avg(raw_col))
        .where(DockerMetric.host_id == host_id, DockerMetric.container_name == name,
               DockerMetric.scraped_at >= since)
        .group_by(rbucket).order_by(rbucket)
    )).all()
    oldest_raw = raw[0][0] if raw else None
    hourly_stmt = (
        select(DockerMetricHourly.bucket, getattr(DockerMetricHourly, metric))
        .where(DockerMetricHourly.host_id == host_id,
               DockerMetricHourly.container_name == name,
               DockerMetricHourly.bucket >= since)
        .order_by(DockerMetricHourly.bucket)
    )
    if oldest_raw is not None:
        hourly_stmt = hourly_stmt.where(DockerMetricHourly.bucket < oldest_raw)
    hourly = (await db.execute(hourly_stmt)).all()
    return [(t, float(v)) for t, v in hourly] + [(t, float(v)) for t, v in raw]
 async def search_logs(db, *, since, host_id: str | None = None, name: str | None = None,
                      text: str = "", stream: str = "", before=None, limit: int = 50) -> list:
    """Retained log lines matching a filter, newest first (limit + 1 fetched by callers
    that need to know whether more exist)."""
    stmt = (select(DockerLog.id, DockerLog.host_id, DockerLog.container_name,
                   DockerLog.ts, DockerLog.stream, DockerLog.line)
            .where(DockerLog.ts >= since))
    if host_id:
        stmt = stmt.where(DockerLog.host_id == host_id)
    if name:
        stmt = stmt.where(DockerLog.container_name == name)
    if stream in ("stdout", "stderr"):
        stmt = stmt.where(DockerLog.stream == stream)
    if text:
        stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(text)}%", escape="\\"))
    if before is not None:
        stmt = stmt.where(DockerLog.ts < before)
    stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
    return list((await db.execute(stmt)).all())
 async def log_context(db, host_id: str, name: str, ts, n: int) -> tuple[list, list]:
    """Up to n lines either side of `ts` in one container's log, each oldest first."""
    cols = (DockerLog.id, DockerLog.ts, DockerLog.stream, DockerLog.line)
    base = select(*cols).where(DockerLog.host_id == host_id, DockerLog.container_name == name)
    before = (await db.execute(
        base.where(DockerLog.ts < ts).order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(n)
    )).all()
    after = (await db.execute(
        base.where(DockerLog.ts > ts).order_by(DockerLog.ts, DockerLog.id).limit(n)
    )).all()
    return list(reversed(before)), list(after)
 async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]:
    """{(host_id, container_name): [die/oom events since `since`, newest first]}."""
    rows = (await db.execute(
        select(DockerEvent)
        .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
        .order_by(DockerEvent.at.desc())
    )).scalars()
    out: dict[tuple[str, str], list] = {}
    for e in rows:
        out.setdefault((e.host_id, e.container_name), []).append(e)
    return out
 async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None,
                         limit: int = 20) -> list[tuple[str, str, datetime]]:
    """(host_id, container_name, last seen) of containers Steward still holds
    logs or events for, newest first — including ones Docker has removed.
    exact: the name itself, from both logs and events. Otherwise names that
    contain `want` (case-insensitive), from events only: a substring match
    over every retained log line's name would scan the log table, and a
    container Steward watched die has a die or stop event.
    """
    seen: dict[tuple[str, str], datetime] = {}
    sources = ((DockerLog, DockerLog.ts), (DockerEvent, DockerEvent.at)) if exact \
        else ((DockerEvent, DockerEvent.at),)
    for model, at in sources:
        match = (model.container_name == want if exact
                 else model.container_name.ilike(f"%{escape_like(want)}%", escape="\\"))
        stmt = (select(model.host_id, model.container_name, func.max(at))
                .where(match).group_by(model.host_id, model.container_name))
        if host_id:
            stmt = stmt.where(model.host_id == host_id)
        for hid, name, last in (await db.execute(stmt)).all():
            key = (hid, name)
            if key not in seen or last > seen[key]:
                seen[key] = last
    rows = sorted(((h, n, t) for (h, n), t in seen.items()), key=lambda r: r[2], reverse=True)
    return rows[:limit]
 async def unhealthy_since(db, keys) -> dict[tuple[str, str], datetime]:
    """{(host_id, container_name): when it last turned unhealthy} for the given
    containers, from their health_change events; absent when none is retained."""
    keys = set(keys)
    if not keys:
        return {}
    rows = (await db.execute(
        select(DockerEvent)
        .where(DockerEvent.event == "health_change",
               DockerEvent.detail.like("%→unhealthy"),
               DockerEvent.container_name.in_({name for _, name in keys}))
        .order_by(DockerEvent.at.desc())
    )).scalars()
    out: dict[tuple[str, str], datetime] = {}
    for e in rows:
        key = (e.host_id, e.container_name)
        if key in keys and key not in out:
            out[key] = e.at
    return out
 PRUNE_TARGETS = ("containers", "images", "system")
 def prune_extra_vars(prune_target: str) -> dict:
    """Extra-vars for the bundled docker_prune playbook. "Prune unused images"
    means ALL unused (docker image prune -a), not just dangling; the full system
    prune stays conservative (-f, no -a) per the operator's choice (m78)."""
    extra_vars: dict = {"prune_target": prune_target}
    if prune_target == "images":
        extra_vars["prune_all_images"] = True
    return extra_vars
@@ -11,28 +11,12 @@ For every swarm service we list the real replicas we have detail for, then add
 "ghost" replicas for the remaining placement count on each node (agent-less
 nodes). Non-swarm containers pass through grouped by host, unchanged.
 A third source, DockerSwarmService.tasks_json (agent ≥ 1.8.0), is Swarm's own
 record of each service's newest tasks. It decides which task containers are
 *superseded* — replaced by Swarm, so neither a replica nor a crash — and gives
 each service its failures across task retries and the error of its last
 failed task. Swarm restarts a task by creating a new one (a new container), so
 failures can only be counted per service, never per container.
 Kept model-free (no ORM imports) so it's unit-testable and safe to import in
 tests without the plugin-loader "table already defined" gotcha.
 """
 from __future__ import annotations
 import json
 from datetime import datetime, timedelta, timezone
 # Task states that mean the task ended in error: `failed` (its container exited
 # non-zero or never started) and `rejected` (Swarm could not start it at all,
 # e.g. no such image, a bad mount).
 FAILED_TASK_STATES = ("failed", "rejected")
 # Desired states of a task Swarm still means to run. Anything else (shutdown,
 # remove, complete, orphaned) is a task Swarm has finished with.
 _CURRENT_DESIRED = ("running", "ready", "")
 def _service_state(running: int, desired: int) -> str:
@@ -52,220 +36,30 @@ def _placement(service) -> list[dict]:
    return [p for p in data if isinstance(p, dict)]
-def parse_tasks(service) -> list[dict]:
+def build_swarm_services(containers, services, node_hostname: dict, host_name: dict) -> dict:
-    """A service row's task records, newest first; [] when its agent sent none."""
+    """Return {"services": [...], "standalone": [...]}.
    raw = getattr(service, "tasks_json", None) or "[]"
    try:
        data = json.loads(raw)
    except (ValueError, TypeError):
        return []
    return [t for t in data if isinstance(t, dict)]
-
+    services[i] = {name, mode, desired, running, state, replicas:[...]}
 def task_time(task: dict) -> datetime | None:
    """A task's status timestamp. Docker writes nanoseconds; Python ≥ 3.11 keeps
    the first six digits."""
    raw = task.get("at") or ""
    try:
        at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
    except ValueError:
        return None
    return at if at.tzinfo else at.replace(tzinfo=timezone.utc)
 def task_failures(tasks: list[dict], since: datetime) -> int:
    """How many tasks ended in error at or after `since`."""
    n = 0
    for t in tasks:
        at = task_time(t)
        if t.get("state") in FAILED_TASK_STATES and at is not None and at >= since:
            n += 1
    return n
 def last_task_error(tasks: list[dict]) -> dict | None:
    """Why the service's newest troubled task is in trouble, from Swarm itself.
    The newest task that failed or was rejected, or that Swarm attached an error
    to while still trying (a pending task with "no suitable node", say). None
    when no retained task has a problem.
    """
    for t in tasks:  # newest first
        if t.get("state") in FAILED_TASK_STATES or t.get("error"):
            return {"state": t.get("state"), "error": t.get("error") or None,
                    "message": t.get("message") or None, "exit_code": t.get("exit_code"),
                    "node_id": t.get("node_id") or None, "at": t.get("at") or None}
    return None
 def task_summary(tasks: list[dict], now: datetime, crash_loop_deaths: int,
                 node_hostname: dict) -> dict:
    """What a service's task records say: tasks_known, failures_1h,
    failures_24h (None without records), crash_looping, and last_error with
    the node's hostname as `node`."""
    known = bool(tasks)
    failures_1h = task_failures(tasks, now - timedelta(hours=1)) if known else None
    last_error = last_task_error(tasks)
    if last_error is not None:
        nid = last_error["node_id"]
        last_error["node"] = node_hostname.get(nid) or (nid[:12] if nid else None)
        at = task_time(last_error)
        last_error["age_s"] = max(0, int((now - at).total_seconds())) if at else None
    return {
        "tasks_known": known,
        "failures_1h": failures_1h,
        "failures_24h": task_failures(tasks, now - timedelta(hours=24)) if known else None,
        "crash_looping": bool(known and failures_1h >= crash_loop_deaths),
        "last_error": last_error,
    }
 def _oldest_failure(tasks: list[dict], since: datetime) -> datetime | None:
    times = [at for t in tasks if t.get("state") in FAILED_TASK_STATES
             and (at := task_time(t)) is not None and at >= since]
    return min(times, default=None)
 def trouble_since(tasks: list[dict], state: str, crash_looping: bool,
                  now: datetime) -> datetime | None:
    """When a service's current trouble began, as far as its retained tasks
    tell; None when it has none, or the tasks cannot say.
    A crash loop dates from its first failure in the last hour, and a healthy
    service that lost tasks from its first failure in the last 24h. A service
    short of replicas dates from its troubled slots (replicated: the slot;
    global: the node): in each slot whose newest task is not running, the
    oldest of its newest troubled tasks (failed, rejected, or carrying an
    error), walking back until a running task interrupts them. Swarm keeps few
    old tasks, so a long outage reads as the oldest one retained: a lower
    bound.
    """
    if crash_looping:
        return _oldest_failure(tasks, now - timedelta(hours=1))
    if state == "healthy":
        return _oldest_failure(tasks, now - timedelta(hours=24))
    by_slot: dict = {}
    for t in tasks:  # newest first, so each slot's list is too
        by_slot.setdefault(t.get("slot") or t.get("node_id"), []).append(t)
    began = []
    for slot_tasks in by_slot.values():
        oldest = None
        for t in slot_tasks:
            if t.get("state") == "running":
                break
            if t.get("state") in FAILED_TASK_STATES or t.get("error"):
                oldest = task_time(t) or oldest
        if oldest is not None:
            began.append(oldest)
    return min(began, default=None)
 def _slot_key(c) -> str:
    """A task container's slot within its service: "<service>.<slot>.<task>" for
    replicated services, "<service>.<node id>.<task>" for global ones."""
    svc, name = c.service_name, c.name or ""
    rest = name[len(svc) + 1:] if name.startswith(svc + ".") else name
    return rest.split(".", 1)[0]
 def superseded_containers(swarm_cs, tasks_by_id: dict[str, dict]) -> set[tuple]:
    """(host_id, name) of task containers Swarm has replaced.
    With the task on record, its desired state decides: Swarm sets a replaced
    or failed task's desired state to shutdown. Without it (an older agent, or a
    task older than the retained history), a container that is not running is
    superseded when another task in the same slot is running or started later.
    """
    out: set[tuple] = set()
    by_slot: dict[tuple, list] = {}
    for c in swarm_cs:
        by_slot.setdefault((c.service_name, _slot_key(c)), []).append(c)
    for c in swarm_cs:
        task = tasks_by_id.get(getattr(c, "task_id", None) or "")
        if task is not None:
            if task.get("desired_state") not in _CURRENT_DESIRED:
                out.add((c.host_id, c.name))
            continue
        if (getattr(c, "status", "") or "").lower() == "running":
            continue
        mine = getattr(c, "started_at", None)
        for other in by_slot[(c.service_name, _slot_key(c))]:
            if other is c:
                continue
            theirs = getattr(other, "started_at", None)
            if ((getattr(other, "status", "") or "").lower() == "running"
                    or (theirs is not None and (mine is None or theirs > mine))):
                out.add((c.host_id, c.name))
                break
    return out
 def _newest_per_name(services) -> dict[str, object]:
    """Every manager reports the same cluster-global service set; keep the most
    recently updated row per name, so a manager on an older agent (no tasks)
    doesn't shadow one that reports them."""
    by_name: dict[str, object] = {}
    for s in services:
        name = getattr(s, "service_name", None)
        if not name:
            continue
        kept = by_name.get(name)
        if kept is None:
            by_name[name] = s
            continue
        a, b = getattr(s, "updated_at", None), getattr(kept, "updated_at", None)
        if a is not None and (b is None or a > b):
            by_name[name] = s
    return by_name
 def replaced_tasks(containers, services) -> set[tuple]:
    """(host_id, name) of the task containers Swarm has replaced, from the
    container rows and the managers' service rows. What the container views use
    to leave history out of their problem lists."""
    swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
    tasks_by_id = {t["task_id"]: t for s in _newest_per_name(services).values()
                   for t in parse_tasks(s) if t.get("task_id")}
    return superseded_containers(swarm_cs, tasks_by_id)
 def build_swarm_services(containers, services, node_hostname: dict, host_name: dict,
                         *, now: datetime | None = None, crash_loop_deaths: int = 3) -> dict:
    """Return {"services": [...], "standalone": [...], "superseded": {(host_id, name)}}.
    services[i] = {name, mode, desired, running, state, replicas:[...],
                   previous_tasks, tasks_known, failures_1h, failures_24h,
                   crash_looping, last_error}
      real replica  = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
-                       health, restart_count}   # current tasks only
+                       health, restart_count}
      ghost replica = {ghost:True, host, count}   # placement with no local row
      previous_tasks = task containers Swarm replaced, left out of replicas
      failures_*    = tasks that failed or were rejected in the window (None
                      without task data); crash_looping = failures_1h ≥ the
                      Settings → Thresholds crash-loop count
      last_error    = last_task_error(), plus `node` (the node's hostname)
      trouble_since = trouble_since(): when the current trouble began, or None
    standalone[i] = {host_id, host, containers:[...]}  # non-swarm, by host
    superseded    = the task containers Swarm replaced, so callers can skip them
    """
    now = now or datetime.now(timezone.utc)
    containers = list(containers)
    swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
    standalone_cs = [c for c in containers if not getattr(c, "service_name", None)]
-    svc_by_name = _newest_per_name(services)
+    # Dedup service rows by name (every manager reports the same cluster-global
-    tasks_by_svc = {name: parse_tasks(s) for name, s in svc_by_name.items()}
+    # set); keep the first — placement is identical across managers.
-    tasks_by_id = {t["task_id"]: t for ts in tasks_by_svc.values() for t in ts
+    svc_by_name: dict[str, object] = {}
-                   if t.get("task_id")}
+    for s in services:
-    superseded = superseded_containers(swarm_cs, tasks_by_id)
+        name = getattr(s, "service_name", None)
        if name and name not in svc_by_name:
            svc_by_name[name] = s
-    # Current task containers grouped by service name; replaced ones are counted.
+    # Real containers grouped by service name.
    cs_by_service: dict[str, list] = {}
    previous: dict[str, int] = {}
    for c in swarm_cs:
        if (c.host_id, c.name) in superseded:
            previous[c.service_name] = previous.get(c.service_name, 0) + 1
            continue
        cs_by_service.setdefault(c.service_name, []).append(c)
    def _host_of(c):
@@ -323,19 +117,13 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
        if desired is None:
            desired = len(replicas)
        tasks = tasks_by_svc.get(name, [])
        state = _service_state(int(running or 0), int(desired or 0))
        summary = task_summary(tasks, now, crash_loop_deaths, node_hostname)
        out_services.append({
            "name": name,
            "mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
            "running": running,
            "desired": desired,
-            "state": state,
+            "state": _service_state(int(running or 0), int(desired or 0)),
            "replicas": replicas,
            "previous_tasks": previous.get(name, 0),
            **summary,
            "trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
        })
    # Non-swarm containers grouped by host, running-first order preserved.
@@ -349,4 +137,4 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
        g["containers"].append(c)
    standalone = sorted(groups.values(), key=lambda g: g["host"].lower())
-    return {"services": out_services, "standalone": standalone, "superseded": superseded}
+    return {"services": out_services, "standalone": standalone}
@@ -1,22 +0,0 @@
 {# docker/_container_logs_lines.html — log-line fragment, HTMX-polled (m79).
   {{ l.line }} is auto-escaped by Jinja, so log content can't inject markup. #}
 {% if lines %}
 {% for l in lines %}
 <div style="display:flex;gap:0.6rem;white-space:pre-wrap;word-break:break-word;">
  <span style="color:var(--text-dim);flex-shrink:0;" title="{{ l.ts }}">{{ l.ts.strftime("%m-%d %H:%M:%S") }}</span>
  <span style="flex-shrink:0;width:3.2rem;color:{% if l.stream == 'stderr' %}var(--red){% else %}var(--text-muted){% endif %};">{{ l.stream }}</span>
  <span style="flex:1;min-width:0;{% if l.stream == 'stderr' %}color:var(--text);{% endif %}">{{ l.line }}</span>
 </div>
 {% endfor %}
 {% else %}
 <div style="color:var(--text-muted);padding:1rem 0.25rem;">
  {% if filtered %}
  No lines match the current filter.
  {% else %}
  No logs collected yet for <code>{{ name }}</code>. Lines appear within a few
  seconds of the container writing to stdout/stderr — unless it's on the log
  exclude list or log collection is turned off in
  <a href="/settings/thresholds/">Settings → Thresholds &amp; Retention</a>.
  {% endif %}
 </div>
 {% endif %}
@@ -1,25 +0,0 @@
 {#- One line on a swarm service's task failures, from Swarm's own task records
    (agent ≥ 1.8.0): a crash loop or the failures of the last 24h, and how the
    newest troubled task ended, in Swarm's words. Expects `s` with state,
    crash_looping, failures_1h, failures_24h and last_error; `style` is
    optional extra CSS for the wrapper. Renders nothing for a healthy service
    with no recent failures. -#}
 {%- set e = s.last_error -%}
 {%- if s.crash_looping or s.failures_24h or (e and s.state != 'healthy') %}
 <div style="display:flex;flex-wrap:wrap;align-items:baseline;gap:0.2rem 0.6rem;font-size:0.78rem;color:var(--text-muted);{{ style or '' }}">
  {% if s.crash_looping %}
  <span style="color:var(--red);font-weight:600;">Crash-looping · {{ s.failures_1h }} tasks failed in the last hour</span>
  {% elif s.failures_24h %}
  <span style="color:var(--orange);">{{ s.failures_24h }} task{{ 's' if s.failures_24h != 1 }} failed in the last 24h</span>
  {% endif %}
  {% if e %}
  <span title="How Swarm recorded the end of this service's newest failed task">
    Last task {{ e.state }}:
    <span style="color:var(--text);font-family:ui-monospace,monospace;overflow-wrap:anywhere;">{{ e.error or e.message or e.state }}</span>
    {%- if e.exit_code %} · exit {{ e.exit_code }}{% endif %}
    {%- if e.node %} · {{ e.node }}{% endif %}
    {%- if e.age_s is not none %} · {{ e.age_s|duration }} ago{% endif %}
  </span>
  {% endif %}
 </div>
 {%- endif %}
@@ -26,7 +26,6 @@
 <div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
  {{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %}
  {% if host %} · on <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %}
  · <a href="/plugins/docker/container/{{ host_id }}/{{ name }}/logs">View logs</a>
 </div>
 {# ── Facts grid ──────────────────────────────────────────────────────────── #}
@@ -1,51 +0,0 @@
 {# docker/container_logs.html — per-container log viewer (m79) #}
 {% extends "base.html" %}
 {% from "_macros.html" import crumbs %}
 {% block title %}Logs — {{ name }} — Docker — Steward{% endblock %}
 {% block breadcrumb %}{{ crumbs([
  ("Docker", "/plugins/docker/"),
  (name, "/plugins/docker/container/" ~ host_id ~ "/" ~ name),
  ("Logs", "")]) }}{% endblock %}
 {% block content %}
 <div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
  <h1 class="page-title" style="margin-bottom:0;">{{ name }}</h1>
  <span style="font-size:0.9rem;color:var(--text-muted);">logs</span>
 </div>
 <div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1rem;">
  Recent lines (newest first) collected by the host agent{% if host %} on
  <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %} —
  updated every few seconds.
  <a href="/plugins/docker/container/{{ host_id }}/{{ name }}">← back to container</a>
 </div>
 {# ── Controls: stream filter + text search (drive the fragment via HTMX) ───── #}
 <form id="log-controls" onsubmit="return false;"
      style="display:flex;gap:0.6rem;align-items:center;flex-wrap:wrap;margin-bottom:0.6rem;">
  <label style="font-size:0.8rem;color:var(--text-muted);display:flex;align-items:center;gap:0.35rem;">
    Stream
    <select name="stream" style="padding:0.25rem 0.4rem;">
      <option value="all">all</option>
      <option value="stdout">stdout</option>
      <option value="stderr">stderr</option>
    </select>
  </label>
  <input type="search" name="q" placeholder="Filter lines…" autocomplete="off"
         aria-label="Filter log lines"
         style="flex:1;min-width:180px;padding:0.3rem 0.5rem;">
 </form>
 <div class="card-flush">
  <div id="log-lines"
       hx-get="/plugins/docker/container/{{ host_id }}/{{ name }}/logs/lines"
       hx-trigger="load, every 5s, change from:#log-controls, keyup changed delay:400ms from:#log-controls"
       hx-include="#log-controls"
       hx-swap="innerHTML"
       role="log" aria-live="polite" tabindex="0"
       style="max-height:70vh;overflow:auto;padding:0.5rem 0.75rem;
              font-family:ui-monospace,monospace;font-size:0.8rem;line-height:1.5;">
    <div style="color:var(--text-muted);">Loading…</div>
  </div>
 </div>
 {% endblock %}
@@ -7,8 +7,8 @@
 <h1 class="page-title" style="margin-bottom:0.4rem;">Image &amp; disk usage</h1>
 <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
-  Reclaimable = space held by images no container references. Admins can prune
+  Reclaimable = space held by images no container references. Cleanup actions
-  per host below; each action runs an audited Ansible playbook on that host.
+  (prune) arrive in a later release — these are read-only figures for now.
 </p>
 {% if host_groups %}
@@ -47,43 +47,6 @@
    </div>
  </div>
  {# ── Cleanup actions (admin only; audited Ansible prune run) ───────────── #}
  {% if session.user_role == 'admin' %}
  <div style="margin-bottom:1rem;">
    {% if ansible_available and g.has_target %}
    <div style="display:flex;flex-wrap:wrap;gap:0.5rem;align-items:center;">
      <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
            data-msg="Remove all STOPPED containers on {{ g.host_name|e }}? This cannot be undone."
            onsubmit="return confirm(this.dataset.msg);">{{ csrf_field() }}
        <input type="hidden" name="target" value="containers">
        <button type="submit" class="btn btn-sm">Prune stopped containers</button>
      </form>
      <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
            data-msg="Remove ALL unused images on {{ g.host_name|e }} (docker image prune -a)? Any image not used by a container is deleted."
            onsubmit="return confirm(this.dataset.msg);">{{ csrf_field() }}
        <input type="hidden" name="target" value="images">
        <button type="submit" class="btn btn-sm">Prune unused images</button>
      </form>
      <form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune" style="margin:0;"
            data-msg="Run a full system prune on {{ g.host_name|e }}? Removes stopped containers, unused networks, dangling images and build cache."
            onsubmit="return confirm(this.dataset.msg);">{{ csrf_field() }}
        <input type="hidden" name="target" value="system">
        <button type="submit" class="btn btn-sm btn-danger">System prune…</button>
      </form>
    </div>
    <div style="font-size:0.72rem;color:var(--text-muted);margin-top:0.4rem;">
      Reclaimed space appears on the next agent sample.
    </div>
    {% elif not ansible_available %}
    <div style="font-size:0.74rem;color:var(--text-muted);">Cleanup needs the Ansible runner (currently unavailable).</div>
    {% else %}
    <div style="font-size:0.74rem;color:var(--text-muted);">
      Link an <a href="/hosts/{{ g.host_id }}">Ansible target</a> to this host to enable prune actions.
    </div>
    {% endif %}
  </div>
  {% endif %}
  {# ── Per-image table ──────────────────────────────────────────────────── #}
  <div class="card-flush">
    <table class="table">
@@ -26,10 +26,9 @@
  </div>
 </div>
-{# ── Swarm services: collapsed per-service, each current replica labelled with
+{# ── Swarm services: collapsed per-service, each replica labelled with its host.
-   its host (tasks Swarm replaced are left out). Ghost replicas (tasks on nodes
+   Ghost replicas (tasks on nodes with no Steward agent) come from the managers'
-   with no Steward agent) come from the managers' placement data so the picture
+   placement data so the picture is cluster-complete. ──────────────────────── #}
   is cluster-complete. A failing service names Swarm's last task error. ───── #}
 {% if swarm_services %}
 <div class="card-flush" style="margin-bottom:1.5rem;">
  <div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
@@ -47,7 +46,6 @@
          {{ s.running }}/{{ s.desired }} running
        </span>
      </div>
      {% with style="padding-left:1.1rem;margin-bottom:0.35rem;" %}{% include "docker/_swarm_failures.html" %}{% endwith %}
      <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(210px,1fr));gap:0.4rem;padding-left:1.1rem;">
        {% for r in s.replicas %}
        {% if r.ghost %}
@@ -22,7 +22,7 @@
    <table class="table">
      <thead>
        <tr>
-          <th>Service</th><th>Mode</th><th>Replicas</th><th>Image</th><th>Placement</th><th>Task failures</th>
+          <th>Service</th><th>Mode</th><th>Replicas</th><th>Image</th><th>Placement</th>
        </tr>
      </thead>
      <tbody>
@@ -39,14 +39,9 @@
          <td style="font-size:0.78rem;color:var(--text-muted);">
            {% for p in s.placement %}{{ p.node }} ({{ p.running }}){% if not loop.last %}, {% endif %}{% else %}—{% endfor %}
          </td>
          <td style="max-width:360px;">
            {% include "docker/_swarm_failures.html" %}
            {%- if not s.tasks_known %}<span style="font-size:0.78rem;color:var(--text-dim);" title="The reporting manager's agent predates task reporting (1.8.0); update it to see task failures.">—</span>
            {%- elif not (s.crash_looping or s.failures_24h or (s.last_error and s.state != 'healthy')) %}<span style="font-size:0.78rem;color:var(--text-dim);">none in 24h</span>{% endif %}
          </td>
        </tr>
        {% else %}
-        <tr><td colspan="6" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No services in this swarm.</td></tr>
+        <tr><td colspan="5" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No services in this swarm.</td></tr>
        {% endfor %}
      </tbody>
    </table>
@@ -8,7 +8,7 @@
    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--green);">{{ running_count }}</span>
    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">running</span>
  </div>
-  <div title="Containers that died or were OOM-killed in the last 24h, plus swarm tasks that failed (a task Swarm replaced is not a failure)">
+  <div title="Distinct containers with a die or OOM event in the last 24h">
    <span style="font-size:1.4rem;font-weight:700;line-height:1;color:{% if failed_24h %}var(--red){% else %}var(--text-muted){% endif %};">{{ failed_24h }}</span>
    <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">failed (24h)</span>
  </div>
@@ -24,7 +24,6 @@
    <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
      <span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
      <span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
      {% if s.crash_looping %}<span style="color:var(--red);font-size:0.7rem;font-weight:600;flex-shrink:0;" title="{{ s.failures_1h }} tasks failed in the last hour{% if s.last_error %}: {{ s.last_error.error or s.last_error.message or s.last_error.state }}{% endif %}">crash-looping</span>{% endif %}
      <span style="color:var(--text-muted);font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;">{{ s.running }}/{{ s.desired }}</span>
    </div>
    <div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">
@@ -21,24 +21,3 @@ def get_scheduled_tasks() -> list:
 def get_blueprint():
    from .routes import host_agent_bp
    return host_agent_bp
 def register_mcp_tools(register) -> None:
    from .mcp_tools import register_tools
    register_tools(register)
 async def mcp_overview(db) -> list[dict]:
    from .mcp_tools import mcp_overview as findings
    return await findings(db)
 def get_alert_metrics() -> dict[str, list[str]]:
    """Host metrics ingest records for alert rules (resource = the host name;
    disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
    comes from the scheduler, not the agent: "agent offline" is > 180."""
    return {"host_agent": [
        "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
        "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m",
        "uptime_secs", "seconds_since_seen",
    ]}
@@ -20,7 +20,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timezone
-AGENT_VERSION = "1.8.0"
+AGENT_VERSION = "1.6.0"
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -34,10 +34,7 @@ class ConfigError(Exception):
 REQUIRED_KEYS = ("url", "token")
 INT_KEYS = ("interval_seconds",)
-LIST_KEYS = ("mounts", "docker_log_exclude")
+LIST_KEYS = ("mounts",)
 # Truthy strings for bool keys; anything else (incl. empty) is False.
 BOOL_KEYS = ("docker_logs_enabled",)
 _BOOL_TRUE = ("1", "true", "yes", "on")
 def read_config(path: str) -> dict:
@@ -61,8 +58,6 @@ def read_config(path: str) -> dict:
                        raise ConfigError(f"{path}:{lineno}: {key} must be int")
                elif key in LIST_KEYS:
                    cfg[key] = [v.strip() for v in value.split(",") if v.strip()]
                elif key in BOOL_KEYS:
                    cfg[key] = value.lower() in _BOOL_TRUE
                else:
                    cfg[key] = value
    except FileNotFoundError:
@@ -74,10 +69,6 @@ def read_config(path: str) -> dict:
    cfg.setdefault("interval_seconds", 30)
    cfg.setdefault("docker_socket", DEFAULT_DOCKER_SOCKET)
    # Container-log collection is on by default (operator preference); a host can
    # opt out with `docker_logs_enabled = false` or thin it with docker_log_exclude.
    cfg.setdefault("docker_logs_enabled", True)
    cfg.setdefault("docker_log_exclude", [])
    return cfg
@@ -412,15 +403,6 @@ def _rates(cur: dict, prev: dict, dt: float) -> dict:
 DOCKER_API_TIMEOUT = 5.0
 # Container-log collection (m79). Logs ride in the same push as metrics, so one
 # interval's batch is capped to keep a chatty container from bloating a POST (and
 # the backoff ring): once the cap is hit a marker line is emitted and collection
 # stops — the deferred lines come on the next interval. On a container's first
 # interval we seed from a short tail, then switch to an incremental since-cursor
 # kept per container in the agent's rate-state.
 DOCKER_LOG_BATCH_MAX_BYTES = 262144   # 256 KiB of log text per push, all containers
 DOCKER_LOG_FIRST_TAIL = 50            # lines seeded on a container's first interval
 def _dechunk(body: bytes) -> bytes:
    """Decode an HTTP/1.1 chunked-transfer body into the raw payload."""
@@ -668,198 +650,6 @@ def collect_docker(socket_path: str) -> list:
        return list(ex.map(lambda c: _collect_one_container(socket_path, c), containers))
 # ─── container logs (m79) ─────────────────────────────────────────────────────
 #
 # The logs endpoint returns a raw multiplexed byte stream, not JSON, so it needs
 # a sibling of _docker_request that hands back bytes. Non-TTY containers frame
 # stdout/stderr with an 8-byte header (stream byte + 4-byte big-endian length);
 # TTY containers emit unframed bytes. We request timestamps=1, so every line is
 # prefixed with an RFC3339Nano timestamp we parse for the since-cursor.
 _LOG_STREAM_NAMES = {0: "stdout", 1: "stdout", 2: "stderr"}
 def _docker_request_raw(socket_path: str, path: str,
                        timeout: float = DOCKER_API_TIMEOUT) -> bytes:
    """GET `path` from the Docker API over the Unix socket; return the raw body.
    Sibling of `_docker_request` for non-JSON endpoints (container logs). Reuses
    the connect / send / de-chunk scaffolding; raises OSError on any transport
    problem or non-2xx status so callers can silent-skip. A 200 with an empty
    body (no new lines since the cursor) returns b"".
    """
    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.settimeout(timeout)
    try:
        sock.connect(socket_path)
        req = (
            "GET " + path + " HTTP/1.1\r\n"
            "Host: docker\r\n"
            "Accept: application/octet-stream\r\n"
            "Connection: close\r\n"
            "\r\n"
        )
        sock.sendall(req.encode("ascii"))
        chunks = []
        while True:
            buf = sock.recv(65536)
            if not buf:
                break
            chunks.append(buf)
    finally:
        sock.close()
    raw = b"".join(chunks)
    head, _, body = raw.partition(b"\r\n\r\n")
    header_text = head.decode("latin-1")
    status_line = header_text.split("\r\n", 1)[0]
    parts = status_line.split(None, 2)
    status = int(parts[1]) if len(parts) >= 2 and parts[1].isdigit() else 0
    if not (200 <= status < 300):
        raise OSError(f"docker API {path} returned {status}")
    if "transfer-encoding: chunked" in header_text.lower():
        body = _dechunk(body)
    return body
 def _demux_docker_logs(raw: bytes) -> list:
    """Split a Docker logs stream into [(stream_name, payload_bytes), …].
    Non-TTY containers multiplex with an 8-byte frame header (stream byte in
    {0,1,2} + 4-byte big-endian length). The stream is treated as framed only if
    the whole buffer parses as clean back-to-back frames; anything else (a TTY
    container's raw stream, or a malformed header) falls back to a single stdout
    blob so no bytes are lost.
    """
    if not raw:
        return []
    frames = []
    i, n = 0, len(raw)
    while i + 8 <= n:
        stream = raw[i]
        if stream > 2 or raw[i + 1] or raw[i + 2] or raw[i + 3]:
            break  # not a valid frame header → not framed
        size = int.from_bytes(raw[i + 4:i + 8], "big")
        if i + 8 + size > n:
            break  # frame overruns the buffer → not framed
        frames.append((_LOG_STREAM_NAMES.get(stream, "stdout"),
                       raw[i + 8:i + 8 + size]))
        i += 8 + size
    if frames and i == n:
        return frames
    return [("stdout", raw)]
 def _parse_log_ts(token: str):
    """Parse a Docker RFC3339Nano timestamp token into a datetime, or None.
    Normalises `Z` → `+00:00` and trims Docker's nanosecond precision to the
    microseconds datetime.fromisoformat accepts (stdlib, Python 3.8+ safe).
    """
    t = token.strip()
    if not t:
        return None
    if t.endswith("Z"):
        t = t[:-1] + "+00:00"
    m = re.match(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(\.\d+)?(.*)$", t)
    if m:
        frac = m.group(2) or ""
        if len(frac) > 7:               # "." + up to 6 fractional digits
            frac = frac[:7]
        t = m.group(1) + frac + m.group(3)
    try:
        return datetime.fromisoformat(t)
    except ValueError:
        return None
 def _parse_container_logs(raw: bytes) -> list:
    """Demux + line-split a logs stream → [(dt|None, stream, line), …] in order.
    Each line is timestamp-prefixed (we request timestamps=1); an unparseable
    prefix yields dt=None with the full raw line kept.
    """
    out = []
    for stream, payload in _demux_docker_logs(raw):
        text = payload.decode("utf-8", "replace")
        for raw_line in text.split("\n"):
            if not raw_line:
                continue
            token, _, msg = raw_line.partition(" ")
            dt = _parse_log_ts(token)
            if dt is not None:
                out.append((dt, stream, msg))
            else:
                out.append((None, stream, raw_line))
    return out
 def collect_docker_logs(socket_path: str, containers: list, state: dict,
                        exclude=None,
                        max_bytes: int = DOCKER_LOG_BATCH_MAX_BYTES) -> list:
    """New container-log lines since the last interval, for running containers.
    Keeps a per-container since-cursor in `state["docker_log_cursors"]`
    (name → unix ts) so each interval fetches only what's new; a container's
    first interval seeds from a short tail. The whole batch is capped at
    `max_bytes` of log text — once exceeded a truncation marker is appended and
    collection stops (logs must never balloon a push). `exclude` container names
    are skipped entirely (a per-host bandwidth opt-out). Best-effort: a
    per-container transport error just contributes nothing this interval.
    Returns [{"container", "stream", "ts", "line"}, …].
    """
    exclude_set = set(exclude or ())
    cursors = state.setdefault("docker_log_cursors", {})
    running = {c.get("name") for c in (containers or [])
               if c.get("name") and c.get("status") == "running"}
    # Forget cursors for containers no longer running so state can't grow without
    # bound over the agent's lifetime.
    for gone in [k for k in cursors if k not in running]:
        del cursors[gone]
    out: list = []
    total = 0
    truncated = False
    for c in (containers or []):
        name = c.get("name")
        if not name or name not in running or name in exclude_set:
            continue
        prev = cursors.get(name)
        base = f"/containers/{name}/logs?stdout=1&stderr=1&timestamps=1"
        # since accepts a fractional Unix ts; we still dedup the boundary line.
        path = (f"{base}&tail={DOCKER_LOG_FIRST_TAIL}" if prev is None
                else f"{base}&since={prev:.9f}")
        try:
            raw = _docker_request_raw(socket_path, path)
        except (OSError, ValueError):
            continue
        newest = prev
        for dt, stream, line in _parse_container_logs(raw):
            ts_unix = dt.timestamp() if dt is not None else None
            if prev is not None and ts_unix is not None and ts_unix <= prev:
                continue  # boundary line already sent last interval
            if total >= max_bytes:
                truncated = True
                break
            out.append({"container": name, "stream": stream,
                        "ts": dt.isoformat() if dt is not None else None,
                        "line": line})
            total += len(line)
            if ts_unix is not None and (newest is None or ts_unix > newest):
                newest = ts_unix
        if newest is not None:
            cursors[name] = newest
        if truncated:
            break
    if truncated:
        out.append({"container": "_steward", "stream": "stderr",
                    "ts": datetime.now(timezone.utc).isoformat(),
                    "line": (f"[steward] log batch truncated at {max_bytes} bytes; "
                             "remaining lines deferred to the next interval")})
    return out
 # ─── swarm (manager-only) ────────────────────────────────────────────────────
@@ -874,46 +664,14 @@ def _swarm_is_manager(info: dict) -> bool:
    return bool((info or {}).get("Swarm", {}).get("ControlAvailable"))
 # Newest tasks kept per service. Swarm retains about 5 per slot by default, so
 # this covers a replicated service's recent history and a crash loop's deaths
 # without shipping a global service's whole fleet every sample.
 SWARM_TASKS_PER_SERVICE = 10
 _TASK_TEXT_CHARS = 300
 def _swarm_task(t: dict) -> dict:
    """One task as Steward stores it: where it ran, what Swarm wants of it,
    what it is doing, and why it stopped (Err/Message, exit code)."""
    status = t.get("Status") or {}
    cstatus = status.get("ContainerStatus") or {}
    exit_code = cstatus.get("ExitCode")
    return {
        "task_id": t.get("ID", "") or "",
        "slot": t.get("Slot"),
        "node_id": t.get("NodeID", "") or "",
        "desired_state": (t.get("DesiredState") or "").lower(),
        "state": (status.get("State") or "").lower(),
        "message": (status.get("Message") or "")[:_TASK_TEXT_CHARS],
        "error": (status.get("Err") or "")[:_TASK_TEXT_CHARS],
        "exit_code": exit_code if isinstance(exit_code, int) else None,
        "at": status.get("Timestamp") or t.get("UpdatedAt") or "",
    }
 def _swarm_services(services: list, tasks: list) -> list:
    """Roll desired-vs-running replicas per service up from the task list.
    Replica health isn't on the service object — it's derived by counting tasks:
    `running` = tasks currently in state running; `desired` = the service's
    configured replica count (replicated) or the count of tasks the scheduler
-    wants running (global, which has no fixed number). Each service also
+    wants running (global, which has no fixed number).
    carries its newest tasks (`tasks`), so Steward can tell a task Swarm
    replaced from one that crashed, and say why the last one failed.
    """
    tasks_by_svc: dict = {}
    for t in (tasks or []):
        if t.get("ServiceID"):
            tasks_by_svc.setdefault(t["ServiceID"], []).append(_swarm_task(t))
    running_by_svc: dict = {}
    desired_by_svc: dict = {}  # only used for global-mode services
    placement: dict = {}       # svc_id -> {node_id: running_count}
@@ -953,8 +711,6 @@ def _swarm_services(services: list, tasks: list) -> list:
            # sees every task, even ones whose containers live on other hosts).
            "placement": [{"node_id": n, "running": cnt}
                          for n, cnt in sorted(placement.get(sid, {}).items())],
            "tasks": sorted(tasks_by_svc.get(sid, []), key=lambda t: t["at"],
                            reverse=True)[:SWARM_TASKS_PER_SERVICE],
        })
    return out
@@ -1096,17 +852,13 @@ class RingBuffer:
 def build_sample(mounts: list[str], state: dict,
-                 docker_socket: str = DEFAULT_DOCKER_SOCKET, *,
+                 docker_socket: str = DEFAULT_DOCKER_SOCKET) -> dict:
                 docker_logs_enabled: bool = True,
                 docker_log_exclude=None) -> dict:
    """Collect one full sample. Partial samples allowed if a collector fails.
    `state` carries the previous network/disk counters + monotonic timestamp so
-    throughput rates can be derived from deltas (and the per-container log
+    throughput rates can be derived from deltas; it is mutated in place.
-    since-cursors); it is mutated in place. `docker_socket` is probed
+    `docker_socket` is probed best-effort — the `docker` key is omitted entirely
-    best-effort — the `docker` key is omitted entirely when no containers are
+    when no containers are found, so non-Docker hosts add nothing to the payload.
    found, so non-Docker hosts add nothing to the payload. `docker_logs_enabled`
    / `docker_log_exclude` gate incremental container-log collection.
    """
    sample: dict = {"ts": datetime.now(timezone.utc).isoformat()}
    try:
@@ -1188,18 +940,6 @@ def build_sample(mounts: list[str], state: dict,
        if disk is not None:
            sample["docker_disk"] = disk
    # Container logs (m79): incremental per-container tail folded into the same
    # push as metrics. Skipped when there are no containers or logging is off;
    # the key is omitted when nothing new arrived this interval.
    if docker and docker_logs_enabled:
        try:
            logs = collect_docker_logs(docker_socket, docker, state,
                                       exclude=docker_log_exclude)
        except Exception:
            logs = []
        if logs:
            sample["docker_logs"] = logs
    return sample
@@ -1218,18 +958,6 @@ def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict:
 BACKOFF_CAP = 300
 def _drop_logs(sample: dict) -> dict:
    """Strip container logs from a sample before it's buffered for retry.
    Logs are the one payload we never carry across a backoff — they'd bloat the
    ring buffer and go stale — while metrics are kept so an outage doesn't lose
    them. The since-cursor already advanced when the logs were collected, so the
    dropped lines are simply not re-sent (accepted loss during an outage).
    """
    sample.pop("docker_logs", None)
    return sample
 def next_backoff(current: int) -> int:
    if current <= 0:
        return 30
@@ -1294,8 +1022,6 @@ def main_loop(conf_path: str) -> int:
    hostname = cfg.get("hostname") or socket.gethostname()
    mounts = cfg.get("mounts") or default_mounts()
    docker_socket = cfg.get("docker_socket") or DEFAULT_DOCKER_SOCKET
    docker_logs_enabled = cfg.get("docker_logs_enabled", True)
    docker_log_exclude = cfg.get("docker_log_exclude") or []
    buffer = RingBuffer(maxlen=20)
    backoff = 0
    # Carries previous net/disk counters + monotonic ts for rate computation.
@@ -1310,17 +1036,13 @@ def main_loop(conf_path: str) -> int:
                cfg = read_config(conf_path)
                mounts = cfg.get("mounts") or default_mounts()
                docker_socket = cfg.get("docker_socket") or DEFAULT_DOCKER_SOCKET
                docker_logs_enabled = cfg.get("docker_logs_enabled", True)
                docker_log_exclude = cfg.get("docker_log_exclude") or []
                metadata = collect_metadata()  # refresh host_ip/distro on reload
                _log("INFO", "config reloaded")
            except ConfigError as e:
                _log("ERROR", f"reload failed: {e}")
            _reload_requested = False
-        sample = build_sample(mounts, rate_state, docker_socket,
+        sample = build_sample(mounts, rate_state, docker_socket)
                              docker_logs_enabled=docker_logs_enabled,
                              docker_log_exclude=docker_log_exclude)
        buffered = buffer.drain()
        payload = build_payload(
            samples=buffered + [sample],
@@ -1339,12 +1061,12 @@ def main_loop(conf_path: str) -> int:
                _log("ERROR", "server rejected payload (400) — dropping sample")
            elif status == 401:
                _log("ERROR", "token rejected (401) — check config + UI")
-                buffer.push(_drop_logs(sample))
+                buffer.push(sample)
            else:
                _log("WARN", f"POST failed (status={status}); buffering")
                for s in buffered:
-                    buffer.push(_drop_logs(s))
+                    buffer.push(s)
-                buffer.push(_drop_logs(sample))
+                buffer.push(sample)
            backoff = next_backoff(backoff)
            sleep_for = backoff
@@ -1,81 +0,0 @@
 """The host_agent plugin's MCP tools and its get_overview findings."""
 from __future__ import annotations
 from datetime import timedelta
 from sqlalchemy import select
 from steward.hosts import service as host_svc
 from steward.mcp._context import mcp_app
 from steward.mcp.tools._shape import age_s, db_session, iso, now_utc
 from steward.mcp.tools.overview import finding
 from steward.models.hosts import Host
 from steward.models.users import UserRole
 from .models import HostAgentRegistration
 # A registration this old that has never checked in is a failed install, not one in progress.
 PENDING_GRACE = timedelta(minutes=30)
 def agent_state(reg, now, stale_after_s: int) -> str:
    """The agent's state: reporting, stale (stopped checking in) or pending (never has)."""
    if reg.last_seen_at is None:
        return "pending"
    return "reporting" if (now - reg.last_seen_at).total_seconds() <= stale_after_s else "stale"
 async def _agents(db):
    return (await db.execute(
        select(HostAgentRegistration, Host)
        .join(Host, Host.id == HostAgentRegistration.host_id)
        .order_by(Host.name)
    )).all()
 async def list_agents() -> dict:
    """Every host agent Steward has registered, with its version and whether it is reporting.
    `state` is "reporting", "stale" (it checked in before but its last
    check-in is older than the stale window, `stale_after_s`) or "pending"
    (registered, never checked in, usually an install that is still running
    or that failed). Each row carries the agent version, kernel, distro, arch
    and the IP the agent reports. get_host shows an agent's latest metrics.
    """
    now = now_utc()
    stale_after = host_svc.agent_stale_after_seconds(mcp_app().config)
    async with db_session() as db:
        rows = await _agents(db)
    agents = [
        {
            "host_id": h.id, "host_name": h.name,
            "state": agent_state(reg, now, stale_after),
            "last_seen_at": iso(reg.last_seen_at), "last_seen_age_s": age_s(reg.last_seen_at, now),
            "agent_version": reg.agent_version, "kernel": reg.kernel, "distro": reg.distro,
            "arch": reg.arch, "reported_ip": reg.host_ip,
            "registered_at": iso(reg.created_at),
        }
        for reg, h in rows
    ]
    counts = {s: sum(1 for a in agents if a["state"] == s) for s in ("reporting", "stale", "pending")}
    return {"stale_after_s": stale_after, "counts": counts, "agents": agents, "total": len(agents)}
 async def mcp_overview(db) -> list[dict]:
    """Agents registered more than PENDING_GRACE ago that have never checked in.
    Agents that stopped reporting are already flagged by the core overview
    from their metrics, so they are not repeated here.
    """
    now = now_utc()
    return [
        finding("warning", "agent_never_reported", h.name,
                "host agent was registered but has never checked in; the install may have failed",
                since=reg.created_at, ids={"host_id": h.id}, next="list_agents()")
        for reg, h in await _agents(db)
        if reg.last_seen_at is None and now - reg.created_at > PENDING_GRACE
    ]
 def register_tools(register) -> None:
    register(list_agents, role=UserRole.viewer, read_only=True)
@@ -9,7 +9,7 @@ from pathlib import Path
 import secrets
 from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for, session
-from steward.auth.middleware import public_route, require_role
+from steward.auth.middleware import require_role
 from steward.models.users import UserRole
 from sqlalchemy import select, func, or_, and_
 from datetime import timedelta
@@ -166,7 +166,6 @@ def _error(status: int, code: str, detail: str | None = None) -> tuple[Any, int]
@host_agent_bp.post("/ingest")
@public_route("the agent authenticates with its bearer token")
 async def ingest():
    auth = request.headers.get("Authorization", "")
    if not auth.startswith("Bearer "):
@@ -201,10 +200,8 @@ async def ingest():
            return _error(401, "invalid_token")
        accepted = 0
        metric_rows: list[PluginMetric] = []
        latest_ts: datetime | None = None
        docker_snapshots: list[tuple[datetime, list]] = []
        docker_log_batches: list[tuple[datetime, list]] = []
        latest_swarm: dict | None = None
        latest_swarm_ts: datetime | None = None
        latest_disk: dict | None = None
@@ -214,15 +211,12 @@ async def ingest():
                recorded_at = _parse_ts(sample["ts"])
            except (KeyError, ValueError):
                continue
-            metric_rows.extend(_expand_sample_to_metrics(sample, host.name, recorded_at))
+            metrics = _expand_sample_to_metrics(sample, host.name, recorded_at)
            for m in metrics:
                session.add(m)
            docker = sample.get("docker")
            if isinstance(docker, list) and docker:
                docker_snapshots.append((recorded_at, docker))
            # Container logs are time-series (append every line, not newest-only);
            # each record carries its own Docker ts, recorded_at is the fallback.
            docker_logs = sample.get("docker_logs")
            if isinstance(docker_logs, list) and docker_logs:
                docker_log_batches.append((recorded_at, docker_logs))
            # Swarm is current-state, not time-series — keep only the newest
            # sample's topology (a manager re-reports it every interval).
            swarm = sample.get("swarm")
@@ -242,16 +236,11 @@ async def ingest():
            await session.rollback()
            return _error(400, "malformed_payload", "no valid samples")
        # Through the alert pipeline, so rules on host metrics can fire.
        from steward.core.alerts import record_metrics
        await record_metrics(session, metric_rows)
        # Hand host-scoped container data to the docker plugin if it's enabled
        # (opportunistic synergy via the capability registry — no hard import,
        # no-op when docker is disabled). A failure here must never sink the
        # whole ingest, so the metrics above still land.
-        if (docker_snapshots or latest_swarm is not None
+        if docker_snapshots or latest_swarm is not None or latest_disk is not None:
                or latest_disk is not None or docker_log_batches):
            from steward.core.capabilities import has_capability, invoke_capability
            if has_capability("docker.persist_host_samples"):
                try:
@@ -261,7 +250,7 @@ async def ingest():
                        await invoke_capability(
                            "docker.persist_host_samples", UserRole.admin,
                            session, host, docker_snapshots, latest_swarm,
-                            latest_disk, docker_log_batches,
+                            latest_disk,
                        )
                except Exception:
                    current_app.logger.exception(
@@ -317,7 +306,6 @@ def _agent_version() -> str:
@host_agent_bp.get("/install.sh")
@public_route("fetched by curl on the host being enrolled; carries an enrolment token")
 async def install_script():
    token = request.args.get("token", "")
    if not token:
@@ -349,7 +337,6 @@ async def install_script():
@host_agent_bp.get("/agent.py")
@public_route("fetched by the installer and by agent self-update; holds no secrets")
 async def agent_source():
    try:
        body = AGENT_SOURCE_PATH.read_text(encoding="utf-8")
@@ -476,7 +463,6 @@ async def _fleet_rows(session) -> list[dict]:
@host_agent_bp.get("/widget")
@require_role(UserRole.viewer)
 async def widget_table():
    """Fleet-glance dashboard widget: one row per monitored host, latest metrics."""
    async with current_app.db_sessionmaker() as session:
@@ -485,7 +471,6 @@ async def widget_table():
@host_agent_bp.get("/widget/history")
@require_role(UserRole.viewer)
 async def widget_history():
    host_id = request.args.get("host_id", "")
    try:
@@ -962,7 +947,6 @@ async def deploy_via_ansible():
        playbook_path="host_agent/install.yml",
        inventory_content=inventory_content,
        inventory_scope=scope,
        redact=list(tokens.values()),
        # extra-vars outrank the playbook's `vars:` defaults; token stays per-host.
        params={"extra_vars_map": {"steward_url": url, "agent_interval": interval}},
        triggered_by=session.get("user_id"),
@@ -1040,7 +1024,6 @@ async def provision_via_ansible():
        playbook_path="host_agent/provision.yml",
        inventory_content=inventory_content,
        inventory_scope=scope,
        redact=list(tokens.values()),
        # extra_vars_map → a JSON -e @file: highest precedence (beats play vars)
        # and space-safe (steward_pubkey carries a comment with a space).
        params={"extra_vars_map": {
@@ -1,12 +1,11 @@
 # plugins/host_agent/scheduler.py
 from __future__ import annotations
 import logging
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import Iterable
 from sqlalchemy import select
 from steward.core.alerts import MetricBatch
 from steward.core.scheduler import ScheduledTask
 from steward.models.hosts import Host
 from .models import HostAgentRegistration
@@ -14,52 +13,66 @@ from .models import HostAgentRegistration
 logger = logging.getLogger(__name__)
-def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
+def _filter_stale(
-                       now: datetime) -> list[tuple[str, float]]:
+    regs: Iterable,
-    """(host name, seconds since its agent last reported) per registration.
+    *,
-
+    now: datetime,
-    An agent that has never reported counts from when it was registered. A
+    stale_after_seconds: int,
-    registration whose host is gone is skipped.
+) -> list:
-    """
+    """Pure staleness filter: returns the subset with last_seen_at strictly
-    out = []
+    older than (now - stale_after_seconds). Rows with last_seen_at=None are
-    for r in regs:
+    never stale (they are unregistered-in-practice)."""
-        name = host_names.get(r.host_id)
+    cutoff = now - timedelta(seconds=stale_after_seconds)
-        seen = r.last_seen_at or r.created_at
+    return [
-        if name is None or seen is None:
+        r for r in regs
-            continue
+        if r.last_seen_at is not None and r.last_seen_at < cutoff
-        out.append((name, max(0.0, (now - seen).total_seconds())))
+    ]
    return out
-async def record_seconds_since_seen(app) -> None:
+async def find_stale_registrations(app, stale_after_seconds: int = 180) -> list[dict]:
    """Record host_agent/<host>/seconds_since_seen for every registered agent,
    so "agent offline" is an ordinary alert rule (seconds_since_seen > 180)."""
    now = datetime.now(timezone.utc)
    async with app.db_sessionmaker() as session:
-        async with session.begin():
+        all_regs = (await session.execute(
-            regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
+            select(HostAgentRegistration)
-            if not regs:
+        )).scalars().all()
-                return
+        stale_rows = _filter_stale(
-            host_names = dict((await session.execute(
+            all_regs,
-                select(Host.id, Host.name).where(Host.id.in_([r.host_id for r in regs]))
+            now=datetime.now(timezone.utc),
-            )).all())
+            stale_after_seconds=stale_after_seconds,
-            batch = MetricBatch()
+        )
-            for name, seconds in seconds_since_seen(regs, host_names, now):
+        if not stale_rows:
-                batch.add(source_module="host_agent", resource_name=name,
+            return []
-                          metric_name="seconds_since_seen", value=seconds, recorded_at=now)
+        hosts = {
-            await batch.record(session)
+            h.id: h for h in (await session.execute(
                select(Host).where(Host.id.in_([r.host_id for r in stale_rows]))
            )).scalars().all()
        }
    return [
        {
            "host_id": r.host_id,
            "host_name": hosts[r.host_id].name if r.host_id in hosts else "?",
            "last_seen_at": r.last_seen_at,
        }
        for r in stale_rows
    ]
 def make_task(app) -> ScheduledTask:
-    async def _record():
+    async def _check_stale():
        try:
-            await record_seconds_since_seen(app)
+            stale = await find_stale_registrations(app)
        except Exception:
-            logger.exception("host_agent: recording seconds_since_seen failed")
+            logger.exception("host_agent stale check failed")
            return
        if stale:
            logger.info(
                "host_agent: %d stale agent(s): %s",
                len(stale),
                [s["host_name"] for s in stale],
            )
    return ScheduledTask(
-        name="host_agent_seconds_since_seen",
+        name="host_agent_stale_check",
-        coro_factory=_record,
+        coro_factory=_check_stale,
        interval_seconds=60,
        run_on_startup=False,
    )
@@ -1,5 +1,22 @@
 {# Current-state fragment for the full-metrics page — polled live via HTMX.
-   Each poll re-renders the latest snapshot the server has. #}
+   Self-contained (defines its own format macros) so each poll re-renders the
   latest snapshot the server has. #}
 {% macro fmt_bps(v) %}
 {%- if v is none -%}—
 {%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s
 {%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s
 {%- else -%}{{ "%.0f"|format(v) }} B/s
 {%- endif -%}
 {% endmacro %}
 {% macro fmt_bytes(v) %}
 {%- if v is none -%}—
 {%- elif v >= 1125899906842624 -%}{{ "%.1f"|format(v / 1125899906842624) }} PB
 {%- elif v >= 1099511627776 -%}{{ "%.1f"|format(v / 1099511627776) }} TB
 {%- elif v >= 1073741824 -%}{{ "%.1f"|format(v / 1073741824) }} GB
 {%- elif v >= 1048576 -%}{{ "%.0f"|format(v / 1048576) }} MB
 {%- else -%}{{ "%.0f"|format(v / 1024) }} KB
 {%- endif -%}
 {% endmacro %}
 {% macro bar(pct) %}
 {%- set p = pct if pct is not none else 0 -%}
 <div style="height:8px;background:var(--bg-elevated);border-radius:4px;overflow:hidden;border:1px solid var(--border);">
@@ -17,7 +34,7 @@
  {% if reg.arch %}<span><span style="color:var(--text-dim);">Arch</span> {{ reg.arch }}</span>{% endif %}
  {% if reg.agent_version %}<span><span style="color:var(--text-dim);">Agent</span> v{{ reg.agent_version }}</span>{% endif %}
  {% set up = hostlvl.get('uptime_secs') %}
-  {% if up %}<span><span style="color:var(--text-dim);">Uptime</span> {{ up|duration }}</span>{% endif %}
+  {% if up %}<span><span style="color:var(--text-dim);">Uptime</span> {{ (up // 86400)|int }}d {{ ((up % 86400) // 3600)|int }}h</span>{% endif %}
  <span><span style="color:var(--text-dim);">Last seen</span> {{ reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") ~ " UTC" if reg.last_seen_at else "never" }}</span>
  {% else %}
  <span>No agent registration found for this host.</span>
@@ -42,7 +59,7 @@
    <span class="stat-val" style="font-size:1.6rem;color:{% if mp is none %}var(--text-dim){% elif mp < 70 %}var(--green){% elif mp < 90 %}var(--yellow){% else %}var(--red){% endif %};">{% if mp is not none %}{{ "%.0f"|format(mp) }}%{% else %}—{% endif %}</span>
    <div style="margin-top:0.5rem;">{{ bar(mp) }}</div>
    <div style="margin-top:0.45rem;font-size:0.72rem;color:var(--text-dim);">
-      avail {{ hostlvl.get('mem_available_bytes')|bytes }} · cache {{ hostlvl.get('mem_cached_bytes')|bytes }} · swap {{ hostlvl.get('swap_used_bytes')|bytes }}
+      avail {{ fmt_bytes(hostlvl.get('mem_available_bytes')) }} · cache {{ fmt_bytes(hostlvl.get('mem_cached_bytes')) }} · swap {{ fmt_bytes(hostlvl.get('swap_used_bytes')) }}
    </div>
  </div>
  <div class="card" style="margin-bottom:0;">
@@ -56,15 +73,15 @@
  <div class="card" style="margin-bottom:0;">
    <div class="section-title" style="margin-bottom:0.35rem;">Network</div>
    <div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
-      <div><span style="color:var(--green);">↓</span> {{ hostlvl.get('net_rx_bps')|rate }}</div>
+      <div><span style="color:var(--green);">↓</span> {{ fmt_bps(hostlvl.get('net_rx_bps')) }}</div>
-      <div><span style="color:var(--red);">↑</span> {{ hostlvl.get('net_tx_bps')|rate }}</div>
+      <div><span style="color:var(--red);">↑</span> {{ fmt_bps(hostlvl.get('net_tx_bps')) }}</div>
    </div>
  </div>
  <div class="card" style="margin-bottom:0;">
    <div class="section-title" style="margin-bottom:0.35rem;">Disk I/O</div>
    <div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
-      <div><span style="color:var(--text-dim);">rd</span> {{ hostlvl.get('disk_read_bps')|rate }}</div>
+      <div><span style="color:var(--text-dim);">rd</span> {{ fmt_bps(hostlvl.get('disk_read_bps')) }}</div>
-      <div><span style="color:var(--text-dim);">wr</span> {{ hostlvl.get('disk_write_bps')|rate }}</div>
+      <div><span style="color:var(--text-dim);">wr</span> {{ fmt_bps(hostlvl.get('disk_write_bps')) }}</div>
    </div>
  </div>
  {% if hostlvl.get('temp_c_max') is not none %}
@@ -111,7 +128,7 @@
  <div style="margin-bottom:0.6rem;">
    <div style="display:flex;justify-content:space-between;font-size:0.8rem;margin-bottom:0.2rem;">
      <span style="font-family:ui-monospace,monospace;">{{ mount }}</span>
-      <span style="color:var(--text-muted);">{{ m.get('disk_used_bytes')|bytes }} / {{ m.get('disk_total_bytes')|bytes }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %}</span>
+      <span style="color:var(--text-muted);">{{ fmt_bytes(m.get('disk_used_bytes')) }} / {{ fmt_bytes(m.get('disk_total_bytes')) }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %}</span>
    </div>
    {{ bar(m.get('disk_used_pct')) }}
  </div>
@@ -128,7 +145,7 @@
    {% for iface, m in nets.items() %}
    <div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
      <span style="font-family:ui-monospace,monospace;">{{ iface }}</span>
-      <span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">↓ {{ m.get('net_rx_bps')|rate }} · ↑ {{ m.get('net_tx_bps')|rate }}</span>
+      <span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">↓ {{ fmt_bps(m.get('net_rx_bps')) }} · ↑ {{ fmt_bps(m.get('net_tx_bps')) }}</span>
    </div>
    {% endfor %}
  </div>
@@ -139,7 +156,7 @@
    {% for dev, m in disks_io.items() %}
    <div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
      <span style="font-family:ui-monospace,monospace;">{{ dev }}</span>
-      <span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">rd {{ m.get('disk_read_bps')|rate }} · wr {{ m.get('disk_write_bps')|rate }}</span>
+      <span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">rd {{ fmt_bps(m.get('disk_read_bps')) }} · wr {{ fmt_bps(m.get('disk_write_bps')) }}</span>
    </div>
    {% endfor %}
  </div>
@@ -45,10 +45,6 @@ cat > "$CONF_FILE" <<EOF
 url = $STEWARD_URL
 token = $AGENT_TOKEN
 interval_seconds = 30
 # Container logs are collected by default. To opt this host out entirely:
 #   docker_logs_enabled = false
 # To skip specific noisy containers (comma-separated names):
 #   docker_log_exclude = watchtower, some-chatty-service
 EOF
 chown "root:$AGENT_USER" "$CONF_FILE"
 chmod 0640 "$CONF_FILE"
@@ -20,17 +20,17 @@
    <a href="/plugins/host_agent/{{ host.id }}/" class="btn btn-sm btn-ghost">Full metrics →</a>
    {% if session.user_role == 'admin' %}
    {% if ansible_available and target %}
-    <form method="post" action="/plugins/host_agent/update" style="margin:0;">{{ csrf_field() }}
+    <form method="post" action="/plugins/host_agent/update" style="margin:0;">
      <input type="hidden" name="inventory_scope" value="{{ scope }}">
      <button type="submit" class="btn btn-sm" title="Refresh agent.py + restart (token preserved)">Update agent</button>
    </form>
    {% endif %}
    <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/rotate-token" style="margin:0;"
-          onsubmit="return confirm('Rotate token? The agent stops reporting until reinstalled/updated with the new token.');">{{ csrf_field() }}
+          onsubmit="return confirm('Rotate token? The agent stops reporting until reinstalled/updated with the new token.');">
      <button type="submit" class="btn btn-sm btn-ghost">Rotate token</button>
    </form>
    <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0;"
-          onsubmit="return confirm('Remove this agent registration? Metrics stop until re-registered.');">{{ csrf_field() }}
+          onsubmit="return confirm('Remove this agent registration? Metrics stop until re-registered.');">
      <button type="submit" class="btn btn-sm btn-danger">Remove</button>
    </form>
    {% endif %}
@@ -49,7 +49,7 @@
      Connects over a one-time bootstrap user + password (not stored).
    </p>
    <form method="post" action="/plugins/host_agent/provision"
-          style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
+          style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;">
      <input type="hidden" name="inventory_scope" value="{{ scope }}">
      <div class="form-group" style="margin:0;">
        <label>Bootstrap user</label>
@@ -96,7 +96,7 @@
              border-radius:6px;padding:0.7rem 0.9rem;display:flex;align-items:center;gap:0.8rem;flex-wrap:wrap;">
    <span style="color:var(--yellow);">⚠</span>
    <span style="font-size:0.84rem;flex:1;min-width:13rem;">No managed SSH key yet — Steward needs one to log into hosts.</span>
-    <form method="post" action="/settings/ansible/generate-key" style="margin:0;">{{ csrf_field() }}
+    <form method="post" action="/settings/ansible/generate-key" style="margin:0;">
      <input type="hidden" name="next" value="/hosts/{{ host.id }}">
      <button type="submit" class="btn btn-sm">Generate managed key</button>
    </form>
@@ -115,7 +115,7 @@
  </p>
  {# Provision: brand-new host (creates steward account + key over a one-time password) #}
  <form method="post" action="/plugins/host_agent/provision"
-        style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;margin-bottom:0.75rem;">{{ csrf_field() }}
+        style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;margin-bottom:0.75rem;">
    <input type="hidden" name="inventory_scope" value="{{ scope }}">
    <div style="font-size:0.78rem;color:var(--text-muted);width:100%;">Provision (first contact — fresh host):</div>
    <div class="form-group" style="margin:0;">
@@ -129,7 +129,7 @@
    <button type="submit" class="btn btn-sm">Provision</button>
  </form>
  {# Install: host already has the steward account (managed key works) #}
-  <form method="post" action="/plugins/host_agent/deploy" style="display:flex;gap:0.6rem;align-items:center;">{{ csrf_field() }}
+  <form method="post" action="/plugins/host_agent/deploy" style="display:flex;gap:0.6rem;align-items:center;">
    <input type="hidden" name="inventory_scope" value="{{ scope }}">
    <span style="font-size:0.78rem;color:var(--text-muted);">Already provisioned?</span>
    <button type="submit" class="btn btn-sm btn-ghost">Install agent (managed key)</button>
@@ -138,7 +138,7 @@
  {% if reg and session.user_role == 'admin' %}
  <form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0.75rem 0 0;"
-        onsubmit="return confirm('Clear this pending registration? Its token is discarded.');">{{ csrf_field() }}
+        onsubmit="return confirm('Clear this pending registration? Its token is discarded.');">
    <button type="submit" class="btn btn-sm btn-ghost">Clear pending registration</button>
  </form>
  {% endif %}
@@ -12,7 +12,7 @@
 </p>
 {% if new_token %}
-<div class="card" style="background:var(--fs-accent-faint);">
+<div class="card" style="background:var(--accent-bg);">
  <h3>New token — copy this install command now</h3>
  <p style="font-size:0.82rem;color:var(--text-muted);">
    This is the only time the raw token is shown. Rotate if you lose it.
@@ -23,7 +23,7 @@
 <div class="card">
  <form method="post" action="/plugins/host_agent/fleet/add-host"
-        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
+        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
    <div class="form-group" style="margin-bottom:0;">
      <label>Host name</label>
      <input type="text" name="name" required>
@@ -63,7 +63,7 @@
    <span style="font-size:0.84rem;flex:1;min-width:14rem;">
      No managed SSH key yet — Steward needs one to log into hosts as <code>steward</code>.
    </span>
-    <form method="post" action="/settings/ansible/generate-key" style="margin:0;">{{ csrf_field() }}
+    <form method="post" action="/settings/ansible/generate-key" style="margin:0;">
      <input type="hidden" name="next" value="/plugins/host_agent/fleet/">
      <button type="submit" class="btn btn-sm">Generate managed key</button>
    </form>
@@ -85,7 +85,7 @@
    agent. Run this once per host.
  </p>
  <form method="post" action="/plugins/host_agent/provision"
-        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
+        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
    <div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
    <div class="form-group" style="margin-bottom:0;">
      <label>Bootstrap user</label>
@@ -111,7 +111,7 @@
    mints a fresh token, and installs the agent. No <code>curl | sh</code> needed.
  </p>
  <form method="post" action="/plugins/host_agent/deploy"
-        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
+        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
    <div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
    <div class="form-group" style="margin-bottom:0;">
      <label>Report interval (s)</label>
@@ -129,7 +129,7 @@
    so identity is preserved.
  </p>
  <form method="post" action="/plugins/host_agent/update"
-        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
+        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
    <div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
    <button type="submit" class="btn">Update</button>
  </form>
@@ -156,12 +156,12 @@
        <td class="td-actions">
          <form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/rotate-token"
                style="display:inline;"
-                onsubmit="return confirm('Rotate the token for this host? The existing agent will stop working until reinstalled with the new token.');">{{ csrf_field() }}
+                onsubmit="return confirm('Rotate the token for this host? The existing agent will stop working until reinstalled with the new token.');">
            <button type="submit" class="btn btn-ghost btn-sm">Rotate token</button>
          </form>
          <form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/delete"
                style="display:inline;"
-                onsubmit="return confirm('Delete this host registration? The agent will be unable to report metrics until re-registered.');">{{ csrf_field() }}
+                onsubmit="return confirm('Delete this host registration? The agent will be unable to report metrics until re-registered.');">
            <button type="submit" class="btn btn-danger btn-sm">Delete</button>
          </form>
        </td>
@@ -22,7 +22,7 @@
    {% endif %}
    <span class="hist-range" style="margin-left:auto;display:inline-flex;border:1px solid var(--border-mid);border-radius:5px;overflow:hidden;">
      {% for opt in [1, 6, 24] %}
-      <button type="button" onclick="histRange({{ wid }}, {{ opt }})"
+      <button type="button" onclick="histRange({{ wid }}, '{{ host.id }}', {{ opt }})"
        style="border:0;cursor:pointer;padding:0.1rem 0.45rem;font-size:0.72rem;
               background:{% if opt == hours %}var(--accent){% else %}transparent{% endif %};
               color:{% if opt == hours %}#fff{% else %}var(--text-muted){% endif %};">{{ opt }}h</button>
@@ -36,15 +36,11 @@
 <script>
 // Live time-range switch. Rewrites the parent cell's hx-get so the choice sticks
 // across polls (htmx re-reads the attribute each poll), then fetches immediately.
-// Only `hours` changes: the rest of the query (host, widget id, a share link's
+window.histRange = function (wid, hostId, hours) {
 // token) is kept, so the switch also works on a shared dashboard.
 window.histRange = function (wid, hours) {
  var cell = document.getElementById("widget-" + wid);
  if (!cell) return;
-  var parts = (cell.getAttribute("hx-get") || "").split("?");
+  var base = (cell.getAttribute("hx-get") || "").split("?")[0];
-  var query = new URLSearchParams(parts[1] || "");
+  var url = base + "?host_id=" + encodeURIComponent(hostId) + "&hours=" + hours + "&wid=" + wid;
  query.set("hours", hours);
  var url = parts[0] + "?" + query.toString();
  cell.setAttribute("hx-get", url);
  if (window.htmx) window.htmx.ajax("GET", url, { target: cell, swap: "innerHTML" });
 };
@@ -13,13 +13,21 @@
  <div style="line-height:0;height:16px;">{{ svg | safe }}</div>
 </div>
 {% endmacro %}
 {# Human-readable throughput, matching the host-detail page. #}
 {% macro fmt_bps(v) %}
 {%- if v is none -%}—
 {%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s
 {%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s
 {%- else -%}{{ "%.0f"|format(v) }} B/s
 {%- endif -%}
 {% endmacro %}
 {# Two-line down/up (or read/write) throughput cell. #}
 {% macro io_cell(label, down_sym, down_val, up_sym, up_val, title="") %}
 <div style="min-width:80px;flex:0 0 auto;" title="{{ title }}">
  <div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
  <div style="font-size:0.72rem;font-variant-numeric:tabular-nums;line-height:1.3;">
-    <div>{{ down_sym }} {{ down_val|rate }}</div>
+    <div>{{ down_sym }} {{ fmt_bps(down_val) }}</div>
-    <div>{{ up_sym }} {{ up_val|rate }}</div>
+    <div>{{ up_sym }} {{ fmt_bps(up_val) }}</div>
  </div>
 </div>
 {% endmacro %}
@@ -26,9 +26,3 @@ def get_blueprint():
 def get_nav() -> list[dict]:
    # Surfaces in the sidebar's Infrastructure group (SNMP device readings).
    return [{"label": "SNMP", "href": "/plugins/snmp/"}]
 def get_alert_metrics() -> dict[str, list[str]]:
    """SNMP metric names are the OID labels each device is configured with, so
    the list is empty here and the rule form offers the labels recorded so far."""
    return {"snmp": []}
@@ -27,7 +27,7 @@ def make_poll_task(app: "Quart") -> ScheduledTask:
 async def _do_poll(app: "Quart") -> None:
    from .poller import poll_device
-    from steward.core.alerts import MetricBatch
+    from steward.core.alerts import record_metric
    devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", [])
    if not devices:
@@ -35,7 +35,6 @@ async def _do_poll(app: "Quart") -> None:
    async with app.db_sessionmaker() as session:
        async with session.begin():
            batch = MetricBatch()
            for device in devices:
                if not isinstance(device, dict):
                    continue
@@ -56,7 +55,8 @@ async def _do_poll(app: "Quart") -> None:
                    continue
                for label, value in readings.items():
-                    batch.add(
+                    await record_metric(
                        session=session,
                        source_module="snmp",
                        resource_name=name,
                        metric_name=label,
@@ -67,4 +67,3 @@ async def _do_poll(app: "Quart") -> None:
                    logger.debug("SNMP polled %s (%s): %d OID(s)", name, host, len(readings))
                else:
                    logger.debug("SNMP polled %s (%s): no readings", name, host)
            await batch.record(session)
@@ -31,7 +31,7 @@
    {% for oid_cfg in device.oids %}
    {% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
    {% set val = device.readings.get(label) %}
-    <div style="background:var(--bg-elevated);border-radius:6px;padding:0.6rem 0.9rem;">
+    <div style="background:var(--bg-alt);border-radius:6px;padding:0.6rem 0.9rem;">
      <div style="font-size:0.72rem;color:var(--text-dim);margin-bottom:0.2rem;font-family:monospace;">{{ label }}</div>
      {% if val is not none %}
      <div style="font-size:1.1rem;font-weight:600;font-variant-numeric:tabular-nums;">
@@ -36,7 +36,7 @@
  {% endif %}
 </div>
 {% if not loop.last %}
-<div style="height:1px;background:var(--border);margin:0.1rem 0;"></div>
+<div style="height:1px;background:var(--border-low);margin:0.1rem 0;"></div>
 {% endif %}
 {% endfor %}
 {% if total > devices|length %}
@@ -33,11 +33,3 @@ def get_blueprint():
 def get_nav() -> list[dict]:
    # Surfaces in the sidebar's Infrastructure group (Traefik services view).
    return [{"label": "Traefik", "href": "/plugins/traefik/"}]
 def get_alert_metrics() -> dict[str, list[str]]:
    """What scheduler.py records for alert rules: per router (resource = the
    router name) and cert_expiry_days per certificate (resource = its CN)."""
    return {"traefik": ["request_rate", "error_rate_4xx_pct", "error_rate_5xx_pct",
                        "latency_p50_ms", "latency_p95_ms", "latency_p99_ms",
                        "response_bytes_rate", "cert_expiry_days"]}
@@ -48,7 +48,7 @@ async def _do_scrape(app: "Quart") -> None:
        extract_certs,
        fetch_metrics,
    )
-    from steward.core.alerts import MetricBatch
+    from steward.core.alerts import record_metric
    metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"]
@@ -71,7 +71,6 @@ async def _do_scrape(app: "Quart") -> None:
    async with app.db_sessionmaker() as session:
        async with session.begin():
            batch = MetricBatch()
            # ── Per-service metrics ───────────────────────────────────────────
            for router_name, metrics in router_metrics.items():
                row = TraefikMetric(
@@ -96,7 +95,8 @@ async def _do_scrape(app: "Quart") -> None:
                    ("latency_p99_ms", metrics["latency_p99_ms"]),
                    ("response_bytes_rate", metrics["response_bytes_rate"]),
                ]:
-                    batch.add(
+                    await record_metric(
                        session=session,
                        source_module="traefik",
                        resource_name=router_name,
                        metric_name=metric_name,
@@ -125,15 +125,14 @@ async def _do_scrape(app: "Quart") -> None:
                # Emit cert_expiry_days for alert rules
                days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0
-                batch.add(
+                await record_metric(
                    session=session,
                    source_module="traefik",
                    resource_name=cert["cn"] or cert["serial"],
                    metric_name="cert_expiry_days",
                    value=max(0.0, days_remaining),
                )
            await batch.record(session)
    logger.debug(
        "Traefik scrape: %d router(s), %d cert(s)",
        len(router_metrics),
@@ -2,7 +2,7 @@
 {# ── Global stats bar ──────────────────────────────────────────────────────── #}
 {% if global_stat %}
-<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--bg-card);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;">
+<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--card-bg);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;">
  {% if global_stat.open_conns_total is not none %}
  <span>
@@ -28,8 +28,9 @@
  {% if global_stat.process_memory_bytes is not none %}
  <span>
    <span style="color:var(--text-muted);">Memory</span>
    {% set mb = (global_stat.process_memory_bytes / 1048576) %}
    <span style="margin-left:0.4rem;">
-      {{ global_stat.process_memory_bytes|bytes }}
+      {% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.0f"|format(mb) }} MB{% endif %}
    </span>
  </span>
  {% endif %}
@@ -57,7 +58,9 @@
            <td style="color:var(--text-muted);padding:0.2rem 0;">Bandwidth</td>
            <td style="padding:0.2rem 0.5rem;">
              {% set bps = r.latest.response_bytes_rate %}
-              {{ bps|rate }}
+              {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
              {% elif bps >= 1024 %}{{ "%.1f"|format(bps / 1024) }} KB/s
              {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
            </td>
            <td style="padding:0.2rem 0;"></td>
        </tr>
@@ -119,6 +122,6 @@
 </div>
 {% else %}
 <div class="card">
-    <p style="color:var(--text-dim);">No Traefik metrics yet. Configure <code>metrics_url</code> in <code>config.yaml</code> under <code>plugins.traefik</code>.</p>
+    <p style="color:#606080;">No Traefik metrics yet. Configure <code>metrics_url</code> in <code>config.yaml</code> under <code>plugins.traefik</code>.</p>
 </div>
 {% endif %}
@@ -2,7 +2,7 @@
 {% if summary %}
 {# ── Overview bar ─────────────────────────────────────────────────────────── #}
-<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--bg-card);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;align-items:center;">
+<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--card-bg);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;align-items:center;">
  <span>
    <span style="color:var(--text-muted);">Total</span>
    <span style="margin-left:0.4rem;font-weight:600;">{{ summary.total }}</span>
@@ -20,7 +20,8 @@
  <span>
    <span style="color:var(--text-muted);">Bytes</span>
    <span style="margin-left:0.4rem;">
-      {{ summary.total_bytes|bytes }}
+      {% set mb = summary.total_bytes / 1048576 %}
      {% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.1f"|format(mb) }} MB{% endif %}
    </span>
  </span>
@@ -26,7 +26,9 @@
      <span style="color:var(--text-muted);">bw</span>
      <span style="margin-left:0.3rem;">
        {% set bps = r.latest.response_bytes_rate %}
-        {{ bps|rate }}
+        {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
        {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
        {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
      </span>
    </span>
    <span title="p95 latency">
@@ -31,9 +31,3 @@ def get_blueprint():
 def get_nav() -> list[dict]:
    # Surfaces in the sidebar's Infrastructure group (UniFi network overview).
    return [{"label": "UniFi", "href": "/plugins/unifi/"}]
 def get_alert_metrics() -> dict[str, list[str]]:
    """What scheduler.py records: the WAN (resource "wan") and the client count
    (resource "clients")."""
    return {"unifi": ["is_up", "latency_ms", "total_clients"]}
@@ -54,7 +54,7 @@ async def _do_poll(app: "Quart") -> None:
    from .client import UnifiClient
    from .models import UnifiClientSnapshot, UnifiDevice, UnifiWanStat
-    from steward.core.alerts import MetricBatch
+    from steward.core.alerts import record_metric
    cfg = app.config["PLUGINS"]["unifi"]
@@ -171,27 +171,28 @@ async def _do_poll(app: "Quart") -> None:
                    ))
            # ── Alert metrics ─────────────────────────────────────────────────
            batch = MetricBatch()
            if wan_stat.latency_ms is not None:
-                batch.add(
+                await record_metric(
                    session=session,
                    source_module="unifi",
                    resource_name="wan",
                    metric_name="latency_ms",
                    value=wan_stat.latency_ms,
                )
-            batch.add(
+            await record_metric(
                session=session,
                source_module="unifi",
                resource_name="wan",
                metric_name="is_up",
                value=1.0 if wan_stat.is_up else 0.0,
            )
-            batch.add(
+            await record_metric(
                session=session,
                source_module="unifi",
                resource_name="clients",
                metric_name="total_clients",
                value=float(len(clients)),
            )
            await batch.record(session)
    logger.debug(
        "UniFi poll: wan=%s, clients=%d, devices=%d",
@@ -21,7 +21,10 @@
      <tr>
        <td style="padding:0.25rem 0.5rem 0.25rem 0;">{{ cat.cat_name or cat.cat_id }}</td>
        <td style="padding:0.25rem 0.3rem;text-align:right;color:var(--text-muted);font-size:0.78rem;white-space:nowrap;">
-          {{ bytes|bytes }}
+          {% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB
          {% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB
          {% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB
          {% else %}{{ bytes }} B{% endif %}
        </td>
        <td style="padding:0.25rem 0;width:4rem;">
          <div style="background:var(--border);border-radius:2px;height:6px;overflow:hidden;">
@@ -47,7 +50,10 @@
        </td>
        <td style="padding:0.25rem 0;text-align:right;font-size:0.78rem;color:var(--text-muted);white-space:nowrap;">
          {% set bytes = c.bytes %}
-          {{ bytes|bytes }}
+          {% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB
          {% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB
          {% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB
          {% else %}{{ bytes }} B{% endif %}
        </td>
      </tr>
      {% endfor %}
@@ -23,7 +23,7 @@
      <th style="text-align:left;font-weight:normal;padding:0 0 0.4rem;">Message</th>
    </tr>
    {% for ev in events %}
-    <tr style="{% if loop.index is divisibleby 2 %}background:var(--fs-table-stripe);{% endif %}">
+    <tr style="{% if loop.index is divisibleby 2 %}background:var(--row-alt,rgba(255,255,255,0.02));{% endif %}">
      <td style="padding:0.25rem 0.5rem 0.25rem 0;color:var(--text-dim);font-size:0.77rem;white-space:nowrap;">
        {{ ev.event_time.strftime("%Y-%m-%d %H:%M:%S") }}
      </td>
@@ -2,7 +2,7 @@
 {# ── WAN health bar ───────────────────────────────────────────────────────── #}
 {% if latest_wan %}
-<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--bg-card);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;align-items:center;">
+<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--card-bg);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;align-items:center;">
  <span>
    <span style="color:var(--text-muted);">WAN</span>
    <span style="margin-left:0.4rem;font-weight:600;color:{% if latest_wan.is_up %}var(--green){% else %}var(--red){% endif %};">
@@ -29,7 +29,9 @@
    <span style="color:var(--text-muted);">&#8595; RX</span>
    <span style="margin-left:0.3rem;">
      {% set bps = latest_wan.rx_bytes_rate %}
-      {{ bps|rate }}
+      {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
      {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s
      {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
    </span>
    <span style="margin-left:0.3rem;">{{ sparkline_rx | safe }}</span>
  </span>
@@ -37,7 +39,9 @@
    <span style="color:var(--text-muted);">&#8593; TX</span>
    <span style="margin-left:0.3rem;">
      {% set bps = latest_wan.tx_bytes_rate %}
-      {{ bps|rate }}
+      {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
      {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s
      {% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
    </span>
    <span style="margin-left:0.3rem;">{{ sparkline_tx | safe }}</span>
  </span>
@@ -45,7 +49,7 @@
  {% if latest_wan.uptime_seconds is not none %}
  <span style="color:var(--text-dim);font-size:0.8rem;">
-    up {{ latest_wan.uptime_seconds|duration }}
+    up {{ (latest_wan.uptime_seconds // 3600) }}h {{ ((latest_wan.uptime_seconds % 3600) // 60) }}m
  </span>
  {% endif %}
 </div>
@@ -92,7 +96,7 @@
      {% if d.state == 1 and d.uptime_seconds %}
      <tr>
        <td colspan="3" style="padding:0 0 0.3rem 1.2rem;color:var(--text-dim);font-size:0.75rem;">
-          up {{ d.uptime_seconds|duration }}
+          up {{ (d.uptime_seconds // 86400) }}d {{ ((d.uptime_seconds % 86400) // 3600) }}h
        </td>
      </tr>
      {% endif %}
@@ -117,9 +121,13 @@
        <td style="text-align:right;padding:0.2rem 0;white-space:nowrap;font-size:0.78rem;">
          {% set rx = c.rx_rate %}{% set tx = c.tx_rate %}
          <span style="color:var(--text-muted);">&#8595;</span>
-          {{ rx|rate }}
+          {% if rx >= 1048576 %}{{ "%.1f"|format(rx/1048576) }}M
          {% elif rx >= 1024 %}{{ "%.0f"|format(rx/1024) }}K
          {% else %}{{ "%.0f"|format(rx) }}B{% endif %}
          <span style="color:var(--text-muted);margin-left:0.3rem;">&#8593;</span>
-          {{ tx|rate }}
+          {% if tx >= 1048576 %}{{ "%.1f"|format(tx/1048576) }}M
          {% elif tx >= 1024 %}{{ "%.0f"|format(tx/1024) }}K
          {% else %}{{ "%.0f"|format(tx) }}B{% endif %}
        </td>
      </tr>
      {% endfor %}
@@ -37,12 +37,16 @@
    <span>
      <span style="color:var(--text-muted);">&#8595;</span>
      {% set bps = latest_wan.rx_bytes_rate %}
-      {{ bps|rate }}
+      {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
      {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
      {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
    </span>
    <span>
      <span style="color:var(--text-muted);">&#8593;</span>
      {% set bps = latest_wan.tx_bytes_rate %}
-      {{ bps|rate }}
+      {% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
      {% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
      {% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
    </span>
    {% endif %}
  </div>
@@ -72,10 +76,14 @@
      {% set total_bps = c.tx_rate + c.rx_rate %}
      <span style="color:var(--text-muted);">&#8595;</span>
      {% set rx = c.rx_rate %}
-      {{ rx|rate }}
+      {% if rx >= 1048576 %}{{ "%.1f"|format(rx / 1048576) }}M
      {% elif rx >= 1024 %}{{ "%.0f"|format(rx / 1024) }}K
      {% else %}{{ "%.0f"|format(rx) }}B{% endif %}
      <span style="color:var(--text-muted);margin-left:0.4rem;">&#8593;</span>
      {% set tx = c.tx_rate %}
-      {{ tx|rate }}
+      {% if tx >= 1048576 %}{{ "%.1f"|format(tx / 1048576) }}M
      {% elif tx >= 1024 %}{{ "%.0f"|format(tx / 1024) }}K
      {% else %}{{ "%.0f"|format(tx) }}B{% endif %}
    </div>
  </div>
  {% endfor %}
@@ -18,9 +18,6 @@ dependencies = [
    "click>=8.0",
    "httpx>=0.27",
    "cryptography>=42",
    # Floor, not a cap: MCPServer (mcp.server.mcpserver) and the mcp.types
    # mirror that steward/mcp/ is written against first ship in SDK 2.2.
    "mcp>=2.2",
 ]
 [project.optional-dependencies]
@@ -1,108 +0,0 @@
 """AI access: personal API tokens for MCP clients.
 Every logged-in user manages their own tokens here, whatever their role — a
 token acts as its user, capped at the role it was given. These routes are
 session-only: a share-link viewer has no user to mint tokens for, and no token
 can reach them (MCP tools never mint or revoke tokens).
 """
 from __future__ import annotations
 from datetime import datetime, timezone
 from quart import Blueprint, abort, current_app, redirect, render_template, request
 from sqlalchemy import select
 from steward.auth.middleware import current_user_id, require_role
 from steward.core import api_tokens
 from steward.core.audit import log_audit
 from steward.core.settings import public_base_url
 from steward.mcp.server import MCP_PATH
 from steward.models.users import User, UserRole
 ai_access_bp = Blueprint("ai_access", __name__, url_prefix="/ai-access")
 _EXPIRY_CHOICES = [("", "Never"), ("30", "30 days"), ("90", "90 days"), ("365", "1 year")]
 def _mcp_url() -> str:
    """The /mcp endpoint as a client outside this request reaches it."""
    return public_base_url(request) + MCP_PATH
 async def _session_user(db) -> User:
    user_id = current_user_id()
    if not user_id:
        abort(403)
    user = (await db.execute(select(User).where(User.id == user_id))).scalar_one_or_none()
    if user is None or not user.is_active:
        abort(403)
    return user
 async def _render_index(db, user: User, *, error: str | None = None, status: int = 200):
    tokens = await api_tokens.list_tokens(db, user.id)
    now = datetime.now(timezone.utc)
    rows = [{"token": t, "state": api_tokens.token_state(t, now)} for t in tokens]
    body = await render_template(
        "ai_access/index.html",
        rows=rows,
        user_role=user.role,
        grantable=api_tokens.grantable_roles(user.role),
        expiry_choices=_EXPIRY_CHOICES,
        error=error,
        mcp_url=_mcp_url(),
    )
    return body, status
@ai_access_bp.get("/")
@require_role(UserRole.viewer)
 async def index():
    async with current_app.db_sessionmaker() as db:
        user = await _session_user(db)
        return await _render_index(db, user)
@ai_access_bp.post("/tokens")
@require_role(UserRole.viewer)
 async def create():
    form = await request.form
    name = form.get("name", "")
    expires = form.get("expires_in_days", "")
    try:
        role = UserRole(form.get("role", UserRole.viewer.value))
    except ValueError:
        role = None
    async with current_app.db_sessionmaker() as db:
        user = await _session_user(db)
        if role is None:
            return await _render_index(db, user, error="Pick a role for the token.", status=400)
        try:
            raw, token = await api_tokens.create_token(
                db, user_id=user.id, user_role=user.role, name=name, role=role,
                expires_in_days=int(expires) if expires.isdigit() and int(expires) > 0 else None,
            )
        except ValueError as exc:
            return await _render_index(db, user, error=str(exc), status=400)
        await db.commit()
    await log_audit(current_app, user.id, user.username, "api_token.create",
                    entity_type="api_token", entity_id=str(token.id),
                    detail={"name": token.name, "role": token.role.value})
    return await render_template("ai_access/created.html", raw_token=raw, token=token,
                                 mcp_url=_mcp_url())
@ai_access_bp.post("/tokens/<int:token_id>/revoke")
@require_role(UserRole.viewer)
 async def revoke(token_id: int):
    async with current_app.db_sessionmaker() as db:
        user = await _session_user(db)
        token = await api_tokens.revoke_token(db, user_id=user.id, token_id=token_id)
        await db.commit()
    if token is None:
        abort(404)
    await log_audit(current_app, user.id, user.username, "api_token.revoke",
                    entity_type="api_token", entity_id=str(token_id),
                    detail={"name": token.name})
    return redirect("/ai-access/")
@@ -4,37 +4,64 @@ from quart import Blueprint, render_template, request, redirect, url_for, curren
 from steward.core.audit import log_audit
 from sqlalchemy import select
 from steward.auth.middleware import require_role
 from steward.alerts import service as alert_svc
 from steward.core.time_range import parse_range
 from steward.ansible import sources as src_module
-from steward.models.alerts import AlertRule, AlertOperator, MaintenanceWindow
+from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertOperator, MaintenanceWindow
 from steward.models.metrics import PluginMetric
 from steward.models.users import UserRole
 alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
 _OPERATORS = [(op.value, op.value) for op in AlertOperator]
-def _rule_form(form) -> dict:
+# Static catalog: source_module → available metric names.
-    """The rule form's fields, as alert_svc.rule_fields takes them."""
+# Unified monitors emit metrics under their TYPE (icmp/tcp/dns/http), each with
-    return {
+# is_up (1.0 up / 0.0 down) and response_ms (where the probe times it).
-        "name": form.get("name", ""),
+METRIC_CATALOG: dict[str, list[str]] = {
-        "source_module": form.get("source_module", ""),
+    "icmp":    ["is_up", "response_ms"],
-        "resource_name": form.get("resource_name", ""),
+    "tcp":     ["is_up", "response_ms"],
-        "metric_name": form.get("metric_name", ""),
+    "dns":     ["is_up", "response_ms"],
-        "operator": form.get("operator", ""),
+    "http":    ["is_up", "response_ms"],
-        "threshold": form.get("threshold"),
+    "traefik": ["request_rate", "error_rate", "latency_p50_ms", "latency_p95_ms",
-        "consecutive_failures_required": form.get("consecutive_failures_required"),
+                "latency_p99_ms", "response_bytes_rate", "cert_expiry_days"],
-        "consecutive_ok_required": form.get("consecutive_ok_required"),
+    "unifi":   ["is_up", "latency_ms", "total_clients"],
-        "notify_no_data": "notify_no_data" in form,
+    # restart_count = cumulative restarts (alert on crash-looping); is_healthy =
-    }
+    # 1.0 healthy / 0.0 unhealthy from the container HEALTHCHECK (alert on <1).
    "docker":  ["cpu_pct", "mem_pct", "restart_count", "is_healthy"],
    "host_agent": [
        "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
        "disk_used_pct_worst", "load_1m", "load_5m", "load_15m", "uptime_secs",
    ],
    # snmp metrics are user-defined OID labels — populated dynamically from plugin_metrics
    "snmp":    [],
 }
 # Ordered list for source module picker
 _SOURCE_MODULES = list(METRIC_CATALOG.keys())
-def _form_time(value) -> datetime:
+async def _get_resources(db, source_module: str) -> list[str]:
-    """A datetime-local field, read as UTC."""
+    """Return sorted distinct resource names for a source module.
-    try:
+
-        return datetime.fromisoformat((value or "").strip()).replace(tzinfo=timezone.utc)
+    For monitor types (icmp/tcp/dns/http), supplements with monitor names so
-    except ValueError:
+    the list is populated even before any metrics have been recorded.
-        raise ValueError(f"{value!r} is not a date and time.") from None
+    """
    result = await db.execute(
        select(PluginMetric.resource_name)
        .where(PluginMetric.source_module == source_module)
        .distinct()
        .order_by(PluginMetric.resource_name)
    )
    resources: set[str] = {r for (r,) in result}
    if source_module in ("icmp", "tcp", "dns", "http"):
        from steward.models.monitors import Monitor
        mon_result = await db.execute(
            select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name)
        )
        for (name,) in mon_result:
            resources.add(name)
    return sorted(resources)
 def _is_admin() -> bool:
@@ -96,56 +123,24 @@ def _parse_ansible_action(form) -> tuple[dict | None, str | None]:
@require_role(UserRole.viewer)
 async def list_alerts():
    async with current_app.db_sessionmaker() as db:
-        active = await alert_svc.active_alerts(db)
+        result = await db.execute(
-        rules = await alert_svc.list_rules(db)
+            select(AlertState, AlertRule)
            .join(AlertRule, AlertState.rule_id == AlertRule.id)
            .where(AlertState.state.in_([
                AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending
            ]))
            .order_by(AlertState.last_transition_at.desc())
        )
        active = result.all()
        rules_result = await db.execute(
            select(AlertRule).order_by(AlertRule.name)
        )
        rules = rules_result.scalars().all()
    return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)
 HISTORY_LIMIT = 200
 def _history_row(event, rule) -> dict:
    """One transition as the history page shows it."""
    from steward.core.alerts import MUTED, RELEASED
    notes, muted_by, released = [], None, False
    for part in (event.note or "").split("; "):
        if part.startswith(MUTED):
            muted_by = part[len(MUTED):].strip() or "a maintenance window"
        elif part == RELEASED:
            released = True
        elif part:
            notes.append(part)
    value = "no data" if event.metric_value is None else f"{event.metric_value:g}"
    return {
        "at": event.transitioned_at, "rule": rule,
        "from_state": event.from_state, "to_state": event.to_state,
        "value": value, "condition": f"{rule.operator.value} {rule.threshold:g}",
        "notes": notes, "muted_by": muted_by, "released": released,
        "deliveries": [(label, sent, error) for label, sent, error in (
            ("email", event.email_sent, event.email_error),
            ("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
        "ansible_run_id": event.ansible_run_id,
    }
@alerts_bp.get("/history")
@require_role(UserRole.viewer)
 async def alert_history():
    """Every alert transition in a range, newest first; one rule's with ?rule=."""
    since, range_key = parse_range(request.args.get("range") or "7d")
    async with current_app.db_sessionmaker() as db:
        rules = await alert_svc.list_rules(db)
        chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
        rows = await alert_svc.history(db, since, chosen.id if chosen else None, HISTORY_LIMIT)
    ctx = {"events": [_history_row(e, r) for e, r in rows[:HISTORY_LIMIT]],
           "truncated": len(rows) > HISTORY_LIMIT, "limit": HISTORY_LIMIT,
           "rules": rules, "chosen": chosen, "current_range": range_key}
    if request.headers.get("HX-Request"):
        return await render_template("alerts/_history_rows.html", **ctx)
    return await render_template("alerts/history.html", **ctx)
@alerts_bp.get("/api/rule_fields")
@require_role(UserRole.viewer)
 async def rule_fields_api():
@@ -153,9 +148,18 @@ async def rule_fields_api():
    current_resource = request.args.get("current_resource", "")
    current_metric = request.args.get("current_metric", "")
    async with current_app.db_sessionmaker() as db:
-        resources = await alert_svc.resources_for(db, source) if source else []
+        resources = await _get_resources(db, source) if source else []
-    async with current_app.db_sessionmaker() as db:
+    metrics = METRIC_CATALOG.get(source, [])
-        metrics = await alert_svc.metrics_for(db, source)
+    # Dynamic sources (e.g. snmp) have empty static catalog — query live metric names
    if source and not metrics:
        async with current_app.db_sessionmaker() as db:
            m_result = await db.execute(
                select(PluginMetric.metric_name)
                .where(PluginMetric.source_module == source)
                .distinct()
                .order_by(PluginMetric.metric_name)
            )
            metrics = [r for (r,) in m_result]
    return await render_template(
        "alerts/_rule_fields.html",
        resources=resources,
@@ -168,17 +172,17 @@ async def rule_fields_api():
@alerts_bp.get("/rules/new")
@require_role(UserRole.operator)
 async def new_rule():
-    first_source = alert_svc.source_modules()[0]
+    first_source = _SOURCE_MODULES[0]
    async with current_app.db_sessionmaker() as db:
-        resources = await alert_svc.resources_for(db, first_source)
+        resources = await _get_resources(db, first_source)
    return await render_template(
        "alerts/rules_form.html",
        rule=None,
        operators=_OPERATORS,
-        source_modules=alert_svc.source_modules(),
+        source_modules=_SOURCE_MODULES,
        initial_source=first_source,
        resources=resources,
-        metrics=alert_svc.metric_catalog().get(first_source, []),
+        metrics=METRIC_CATALOG.get(first_source, []),
        ansible_sources=_ansible_source_names(),
    )
@@ -191,15 +195,15 @@ async def edit_rule(rule_id: str):
        rule = result.scalar_one_or_none()
        if rule is None:
            return "Not found", 404
-        resources = await alert_svc.resources_for(db, rule.source_module)
+        resources = await _get_resources(db, rule.source_module)
    return await render_template(
        "alerts/rules_form.html",
        rule=rule,
        operators=_OPERATORS,
-        source_modules=alert_svc.source_modules(),
+        source_modules=_SOURCE_MODULES,
        initial_source=rule.source_module,
        resources=resources,
-        metrics=alert_svc.metric_catalog().get(rule.source_module, []),
+        metrics=METRIC_CATALOG.get(rule.source_module, []),
        ansible_sources=_ansible_source_names(),
    )
@@ -209,20 +213,36 @@ async def edit_rule(rule_id: str):
 async def create_rule():
    form = await request.form
    user_id = session.get("user_id")
    now = datetime.now(timezone.utc)
    # Ansible action is admin-only; non-admins simply can't set one.
    ansible_action = None
    if _is_admin():
        ansible_action, err = _parse_ansible_action(form)
        if err:
            return err, 400
    rule = AlertRule(
        name=form["name"].strip(),
        source_module=form["source_module"].strip(),
        resource_name=form["resource_name"].strip(),
        metric_name=form["metric_name"].strip(),
        operator=AlertOperator(form["operator"]),
        threshold=float(form["threshold"]),
        consecutive_failures_required=int(form.get("consecutive_failures_required") or 1),
        enabled=True,
        ansible_action=ansible_action,
        created_by=user_id,
    )
    state = AlertState(
        rule_id=rule.id,
        state=AlertStateEnum.inactive,
        last_transition_at=now,
    )
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
-            try:
+            db.add(rule)
-                fields = await alert_svc.rule_fields(db, **_rule_form(form))
+            await db.flush()
-            except ValueError as exc:
+            state.rule_id = rule.id
-                return str(exc), 400
+            db.add(state)
            rule = await alert_svc.add_rule(db, fields, created_by=user_id,
                                            ansible_action=ansible_action)
    await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created",
                    entity_type="alert_rule", entity_id=rule.name,
                    detail={"source": rule.source_module, "resource": rule.resource_name,
@@ -247,11 +267,13 @@ async def update_rule(rule_id: str):
            rule = result.scalar_one_or_none()
            if rule is None:
                return "Not found", 404
-            try:
+            rule.name = form["name"].strip()
-                fields = await alert_svc.rule_fields(db, **_rule_form(form))
+            rule.source_module = form["source_module"].strip()
-            except ValueError as exc:
+            rule.resource_name = form["resource_name"].strip()
-                return str(exc), 400
+            rule.metric_name = form["metric_name"].strip()
-            await alert_svc.apply_rule_change(db, rule, fields)
+            rule.operator = AlertOperator(form["operator"])
            rule.threshold = float(form["threshold"])
            rule.consecutive_failures_required = int(form.get("consecutive_failures_required") or 1)
            if apply_action:
                rule.ansible_action = ansible_action
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
@@ -268,7 +290,7 @@ async def toggle_rule(rule_id: str):
            rule = result.scalar_one_or_none()
            if rule is None:
                return "Not found", 404
-            await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled)
+            rule.enabled = not rule.enabled
    action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    action, entity_type="alert_rule", entity_id=rule.name)
@@ -297,12 +319,15 @@ async def delete_rule(rule_id: str):
 async def list_maintenance():
    now = datetime.now(timezone.utc)
    async with current_app.db_sessionmaker() as db:
-        windows = await alert_svc.list_maintenance_windows(db)
+        result = await db.execute(
            select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
        )
        windows = result.scalars().all()
    return await render_template(
        "alerts/maintenance.html",
        windows=windows,
        now=now,
-        source_modules=alert_svc.source_modules(),
+        source_modules=_SOURCE_MODULES,
    )
@@ -312,7 +337,7 @@ async def new_maintenance():
    return await render_template(
        "alerts/maintenance_form.html",
        window=None,
-        source_modules=alert_svc.source_modules(),
+        source_modules=_SOURCE_MODULES,
    )
@@ -321,18 +346,26 @@ async def new_maintenance():
 async def create_maintenance():
    form = await request.form
    user_id = session.get("user_id")
-    try:
+    start_at = datetime.fromisoformat(form["start_at"]).replace(tzinfo=timezone.utc)
-        fields = alert_svc.window_fields(
+    end_at = datetime.fromisoformat(form["end_at"]).replace(tzinfo=timezone.utc)
-            name=form.get("name", ""),
+    scope_source = form.get("scope_source", "").strip() or None
-            start_at=_form_time(form.get("start_at")), end_at=_form_time(form.get("end_at")),
+    scope_resource = form.get("scope_resource", "").strip() or None
-            scope_source=form.get("scope_source"), scope_resource=form.get("scope_resource"))
+    if scope_source is None:
-    except ValueError as exc:
+        scope_resource = None  # can't have resource without source
-        return str(exc), 400
+    window = MaintenanceWindow(
-    window = MaintenanceWindow(created_by=user_id, **fields)
+        name=form["name"].strip(),
        start_at=start_at,
        end_at=end_at,
        scope_source=scope_source,
        scope_resource=scope_resource,
        created_by=user_id,
    )
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            db.add(window)
-    scope_desc = alert_svc.window_scope(window)
+    scope_desc = "all" if not scope_source else (
        f"{scope_source}/{scope_resource}" if scope_resource else f"{scope_source}/*"
    )
    await log_audit(current_app, user_id, session.get("username", ""),
                    "maintenance_window.created", entity_type="maintenance_window",
                    entity_id=window.name, detail={"scope": scope_desc})
@@ -363,12 +396,19 @@ async def delete_maintenance(window_id: str):
@require_role(UserRole.operator)
 async def acknowledge(rule_id: str):
    user_id = session.get("user_id")
    now = datetime.now(timezone.utc)
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
-            try:
+            result = await db.execute(
-                await alert_svc.acknowledge(db, rule_id, user_id)
+                select(AlertState).where(AlertState.rule_id == rule_id)
-            except LookupError:
+            )
            state = result.scalar_one_or_none()
            if state is None:
                return "Not found", 404
-            except ValueError as exc:
+            if state.state != AlertStateEnum.firing:
-                return f"Conflict: {exc}", 409
+                return "Conflict: alert is not in FIRING state", 409
            state.state = AlertStateEnum.acknowledged
            state.acknowledged_by = user_id
            state.acknowledged_at = now
            state.last_transition_at = now
    return redirect(url_for("alerts.list_alerts"))
@@ -1,289 +0,0 @@
 """Alert queries and validation shared by the alerts pages and the MCP tools.
 Each function takes an open session (or plain values), with no Quart request
 context, so the routes and the /mcp tools (which run outside Quart's
 pipeline) call the same code and refuse the same input for the same reasons.
 """
 from __future__ import annotations
 import logging
 from datetime import datetime, timezone
 from sqlalchemy import or_, select
 from steward.models.alerts import (
    AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
 )
 from steward.models.metrics import PluginMetric
 from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
 logger = logging.getLogger(__name__)
 # What core itself records. Unified monitors emit metrics under their TYPE
 # (icmp/tcp/dns/http), each with is_up (1.0 up / 0.0 down) and response_ms
 # (where the probe times it); "steward" is the app watching itself.
 CORE_METRICS: dict[str, list[str]] = {
    "icmp":    ["is_up", "response_ms"],
    "tcp":     ["is_up", "response_ms"],
    "dns":     ["is_up", "response_ms"],
    "http":    ["is_up", "response_ms"],
    "steward": ["open_fds", "open_fds_pct"],
 }
 def metric_catalog() -> dict[str, list[str]]:
    """Every source an alert rule can watch, and its metric names.
    Core's own sources, then each loaded plugin's get_alert_metrics() export
    ({source: [metric names]}), so the catalog is written by whatever emits
    the metrics and a disabled plugin's sources drop out. An empty list means
    the names are dynamic (snmp: the OID labels recorded so far).
    """
    from steward.core.plugin_manager import get_loaded_plugin_modules
    catalog = {src: list(names) for src, names in CORE_METRICS.items()}
    for name, module in get_loaded_plugin_modules().items():
        hook = getattr(module, "get_alert_metrics", None)
        if hook is None:
            continue
        try:
            contributed = hook() or {}
        except Exception:
            logger.exception("Plugin %r: get_alert_metrics() raised; its metrics are skipped", name)
            continue
        for src, names in contributed.items():
            catalog[str(src)] = [str(n) for n in names]
    return catalog
 def source_modules() -> list[str]:
    """The sources in picker order: core's first, then plugins'."""
    return list(metric_catalog())
 # States that need someone's attention: pending (breaching, not yet for long
 # enough), firing, and acknowledged-but-still-breaching.
 ACTIVE_STATES = (AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending)
 async def active_alerts(db) -> list:
    """(AlertState, AlertRule) rows that need attention, latest transition
    first: not yet resolved, or with a series that has gone silent (no_data)."""
    return list((await db.execute(
        select(AlertState, AlertRule)
        .join(AlertRule, AlertState.rule_id == AlertRule.id)
        .where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
        .order_by(AlertState.last_transition_at.desc())
    )).all())
 def alert_label(state: AlertState) -> str:
    """What an alert is, in one word: its state, or "no_data" for a quiet rule
    whose series has gone silent (a firing alert that goes silent stays
    "firing"; check no_data_since for the mark)."""
    if state.no_data_since is not None and state.state == AlertStateEnum.inactive:
        return "no_data"
    return state.state.value
 async def list_rules(db) -> list[AlertRule]:
    return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())
 async def history(db, since: datetime, rule_id: str | None = None,
                  limit: int = 200) -> list[tuple[AlertEvent, AlertRule]]:
    """Alert transitions since `since`, newest first, with their rule; one
    rule's when `rule_id` is given. Returns up to limit+1 rows so a caller can
    tell the list was cut."""
    stmt = (select(AlertEvent, AlertRule)
            .join(AlertRule, AlertEvent.rule_id == AlertRule.id)
            .where(AlertEvent.transitioned_at >= since)
            .order_by(AlertEvent.transitioned_at.desc())
            .limit(limit + 1))
    if rule_id:
        stmt = stmt.where(AlertEvent.rule_id == rule_id)
    return [tuple(row) for row in (await db.execute(stmt)).all()]
 async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
    """Every window, latest start first."""
    return list((await db.execute(
        select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
    )).scalars())
 async def resources_for(db, source_module: str) -> list[str]:
    """Sorted distinct resource names for a source module.
    For monitor types (icmp/tcp/dns/http), supplements with monitor names so
    the list is populated even before any metrics have been recorded.
    """
    result = await db.execute(
        select(PluginMetric.resource_name)
        .where(PluginMetric.source_module == source_module)
        .distinct()
        .order_by(PluginMetric.resource_name)
    )
    resources: set[str] = {r for (r,) in result}
    if source_module in MONITOR_TYPE_VALUES:
        mon_result = await db.execute(
            select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name))
        resources.update(name for (name,) in mon_result)
    return sorted(resources)
 async def metrics_for(db, source_module: str) -> list[str]:
    """The metric names a rule on this source can watch: the static catalog, or
    for dynamic sources (snmp) the names recorded so far."""
    metrics = metric_catalog().get(source_module, [])
    if source_module and not metrics:
        result = await db.execute(
            select(PluginMetric.metric_name)
            .where(PluginMetric.source_module == source_module)
            .distinct()
            .order_by(PluginMetric.metric_name))
        metrics = [r for (r,) in result]
    return metrics
 def _choices(values: list[str], limit: int = 20) -> str:
    if not values:
        return "none yet"
    shown = ", ".join(values[:limit])
    return shown + (f" (and {len(values) - limit} more)" if len(values) > limit else "")
 async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
                      operator: str, threshold, consecutive_failures_required=1,
                      consecutive_ok_required=1, notify_no_data=True) -> dict:
    """Validated column values for a new or edited alert rule.
    The same choices the rule form offers: a known source, a resource that
    source has reported (or, for monitors, a monitor of that type), and a
    metric it emits. ValueError says what was wrong and what is valid.
    """
    name = (name or "").strip()
    if not name:
        raise ValueError("an alert rule needs a name.")
    source_module = (source_module or "").strip()
    catalog = metric_catalog()
    if source_module not in catalog:
        raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
    resource_name = (resource_name or "").strip()
    resources = await resources_for(db, source_module)
    if resource_name not in resources:
        raise ValueError(f"{source_module} has no resource {resource_name!r}. "
                         f"Known: {_choices(resources)}.")
    metric_name = (metric_name or "").strip()
    metrics = await metrics_for(db, source_module)
    if metric_name not in metrics:
        raise ValueError(f"{source_module} does not report {metric_name!r}. "
                         f"Its metrics: {_choices(metrics)}.")
    try:
        op = AlertOperator((operator or "").strip())
    except ValueError:
        raise ValueError(f"operator {operator!r} is not one of "
                         f"{', '.join(o.value for o in AlertOperator)}.") from None
    try:
        threshold = float(threshold)
    except (TypeError, ValueError):
        raise ValueError(f"threshold must be a number, got {threshold!r}.") from None
    try:
        consecutive = (1 if consecutive_failures_required in (None, "")
                       else int(consecutive_failures_required))
    except (TypeError, ValueError):
        raise ValueError("consecutive checks must be a whole number.") from None
    if consecutive < 1:
        raise ValueError("consecutive checks must be at least 1.")
    try:
        ok_checks = (1 if consecutive_ok_required in (None, "")
                     else int(consecutive_ok_required))
    except (TypeError, ValueError):
        raise ValueError("checks to clear must be a whole number.") from None
    if ok_checks < 1:
        raise ValueError("checks to clear must be at least 1.")
    return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
            "metric_name": metric_name, "operator": op, "threshold": threshold,
            "consecutive_failures_required": consecutive,
            "consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)}
 # Changing any of these changes what the rule is watching, so whatever state
 # it had built up no longer means anything.
 WATCHED_FIELDS = ("source_module", "resource_name", "metric_name", "operator", "threshold")
 async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool | None = None) -> None:
    """Apply validated fields (and optionally enabled) to a rule, in the
    caller's transaction. A change to what it watches, or disabling or
    re-enabling it, puts its alert back to inactive and starts its silence
    watch over (core.alerts.reset_rule_state)."""
    from steward.core.alerts import reset_rule_state
    changed = [k for k in WATCHED_FIELDS if k in fields and getattr(rule, k) != fields[k]]
    for k, v in fields.items():
        setattr(rule, k, v)
    disabling = enabled is False and rule.enabled
    enabling = enabled is True and not rule.enabled
    if enabled is not None:
        rule.enabled = bool(enabled)
    if disabling:
        await reset_rule_state(db, rule, "rule disabled")
    elif enabling:
        await reset_rule_state(db, rule, "rule enabled")
    elif changed:
        await reset_rule_state(db, rule, "rule changed: " + ", ".join(changed))
 async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
                   ansible_action: dict | None = None) -> AlertRule:
    """Insert a rule and its inactive state row, in the caller's transaction."""
    rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
    db.add(rule)
    await db.flush()
    db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
                      last_transition_at=datetime.now(timezone.utc)))
    return rule
 async def acknowledge(db, rule_id: str, user_id: str) -> AlertState:
    """Acknowledge a firing alert, in the caller's transaction.
    LookupError when the rule has no state; ValueError when it is not firing.
    """
    state = (await db.execute(
        select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one_or_none()
    if state is None:
        raise LookupError(rule_id)
    if state.state != AlertStateEnum.firing:
        raise ValueError(f"the alert is {state.state.value}, not firing; only a firing alert "
                         "can be acknowledged.")
    now = datetime.now(timezone.utc)
    state.state = AlertStateEnum.acknowledged
    state.acknowledged_by = user_id
    state.acknowledged_at = now
    state.last_transition_at = now
    return state
 def window_fields(*, name: str, start_at: datetime, end_at: datetime,
                  scope_source: str | None, scope_resource: str | None) -> dict:
    """Validated columns for a maintenance window. A resource needs a source."""
    name = (name or "").strip()
    if not name:
        raise ValueError("a maintenance window needs a name.")
    if end_at <= start_at:
        raise ValueError("the window must end after it starts.")
    scope_source = (scope_source or "").strip() or None
    scope_resource = (scope_resource or "").strip() or None
    catalog = metric_catalog()
    if scope_source is not None and scope_source not in catalog:
        raise ValueError(f"source {scope_source!r} is not one of {', '.join(catalog)}.")
    if scope_source is None:
        scope_resource = None  # can't have resource without source
    return {"name": name[:128], "start_at": start_at, "end_at": end_at,
            "scope_source": scope_source, "scope_resource": scope_resource}
 def window_scope(w) -> str:
    if not w.scope_source:
        return "all"
    return f"{w.scope_source}/{w.scope_resource}" if w.scope_resource else f"{w.scope_source}/*"
@@ -61,8 +61,6 @@
      notify: restart steward-agent
    - name: Write agent config
      # The content carries steward_token; --diff and a failure would print it.
      no_log: true
      ansible.builtin.copy:
        dest: /etc/steward-agent.conf
        owner: root
@@ -117,8 +117,6 @@
      notify: restart steward-agent
    - name: Write agent config
      # The content carries steward_token; --diff and a failure would print it.
      no_log: true
      ansible.builtin.copy:
        dest: /etc/steward-agent.conf
        owner: root
@@ -1,64 +1,29 @@
 ---
-# description: Reclaim disk on Docker / Swarm nodes — prune stopped containers, unused images, or a full system prune.
+# description: Reclaim disk on Docker / Swarm nodes by pruning unused images, containers, networks and build cache.
 # steward:category: maintenance
 # steward:confirm: true
 # Reclaim disk on Docker / Docker Swarm nodes by removing unused data.
-# `prune_target` selects the scope (default `system` preserves the original
+# Safe by default: prunes dangling images, stopped containers, unused networks
-# behavior for existing manual/scheduled callers that don't set it):
+# and build cache. Set extra-vars to widen scope:
-#   prune_target=containers   remove stopped containers only (docker container prune)
+#   prune_all_images=true   also remove ALL unused images (not just dangling)
-#   prune_target=images       remove unused images (docker image prune;
+#   prune_volumes=true      also remove unused named volumes (data loss risk)
-#                             + prune_all_images=true → -a, i.e. ALL unused, not just dangling)
+- name: Docker system prune
 #   prune_target=system       docker system prune (dangling images, stopped
 #                             containers, unused networks, build cache). Widen with:
 #                               prune_all_images=true   also ALL unused images
 #                               prune_volumes=true       also unused named volumes (data loss risk)
 - name: Docker prune
  hosts: all
  gather_facts: false
  become: true
  vars:
    prune_target: system
    prune_all_images: false
    prune_volumes: false
  tasks:
    - name: Validate prune_target
      ansible.builtin.assert:
        that: prune_target in ['containers', 'images', 'system']
        fail_msg: "prune_target must be one of: containers, images, system (got '{{ prune_target }}')"
        quiet: true
    # ── Stopped containers only ───────────────────────────────────────────────
    - name: Prune stopped containers
      ansible.builtin.command:
        argv: ['docker', 'container', 'prune', '-f']
      register: container_prune
      changed_when: "'Total reclaimed space: 0B' not in container_prune.stdout"
      when: prune_target == 'containers'
    # ── Unused images (dangling, or all unused with -a) ───────────────────────
    - name: Prune unused images
      ansible.builtin.command:
        argv: >-
          {{ ['docker', 'image', 'prune', '-f']
             + (['-a'] if prune_all_images | bool else []) }}
      register: image_prune
      changed_when: "'Total reclaimed space: 0B' not in image_prune.stdout"
      when: prune_target == 'images'
    # ── Full system prune (default) ───────────────────────────────────────────
    - name: Run docker system prune
      ansible.builtin.command:
        argv: >-
          {{ ['docker', 'system', 'prune', '-f']
             + (['-a'] if prune_all_images | bool else [])
             + (['--volumes'] if prune_volumes | bool else []) }}
-      register: system_prune
+      register: prune_result
-      changed_when: "'Total reclaimed space: 0B' not in system_prune.stdout"
+      changed_when: "'Total reclaimed space: 0B' not in prune_result.stdout"
      when: prune_target == 'system'
    - name: Report reclaimed space
      ansible.builtin.debug:
-        msg: >-
+        msg: "{{ prune_result.stdout_lines | select | list }}"
          {{ ((container_prune.stdout_lines | default([]))
              + (image_prune.stdout_lines | default([]))
              + (system_prune.stdout_lines | default([]))) | select | list }}
@@ -293,45 +293,6 @@ def cancel_run(run_id: str) -> bool:
 # ── Run execution ─────────────────────────────────────────────────────────────
 # ── Secret scrubbing ──────────────────────────────────────────────────────────
 # Ansible echoes values back: a failed task prints its args, -v prints vars,
 # --diff prints templated files. Each line is masked before it is broadcast,
 # written to the log artifact or stored on the run, so a secret the run was
 # handed never leaves it.
 MASK = "***"
 _MIN_SECRET_LEN = 4  # masking "on" or "1" would mangle the output and hide nothing
 def _strings(value) -> list[str]:
    if isinstance(value, str):
        return [value]
    if isinstance(value, dict):
        return [s for v in value.values() for s in _strings(v)]
    if isinstance(value, (list, tuple, set)):
        return [s for v in value for s in _strings(v)]
    return []
 def run_secrets(creds: dict | None, connection: dict | None,
                secret_vars: dict | None, redact: list[str] | None) -> list[str]:
    """Every secret value this run is handed: the global become/vault
    passwords, the bootstrap passwords, the sensitive run variables and any
    the caller names (a per-host agent token)."""
    creds, connection = creds or {}, connection or {}
    values = [creds.get("become_password"), creds.get("vault_password"),
              connection.get("password"), connection.get("become_password"),
              *_strings(secret_vars), *(redact or [])]
    found = {v for v in values if isinstance(v, str) and len(v) >= _MIN_SECRET_LEN}
    return sorted(found, key=len, reverse=True)  # longest first: no partial masks
 def scrub(line: str, secrets: list[str]) -> str:
    for value in secrets:
        if value in line:
            line = line.replace(value, MASK)
    return line
 async def start_run(
    app: "Quart",
    run_id: str,
@@ -342,7 +303,6 @@ async def start_run(
    inventory_content: str | None = None,
    connection: dict | None = None,
    secret_vars: dict | None = None,
    redact: list[str] | None = None,
 ) -> None:
    """Execute ansible-playbook as a subprocess and update the DB run row.
@@ -355,12 +315,10 @@ async def start_run(
    never persisted on the AnsibleRun row, never placed on argv, and not logged.
    secret_vars are run-time playbook variables flagged sensitive; like
    connection they reach the subprocess (merged into the extra-vars file) but
-    are never persisted. Every value in run_secrets() — redact names more — is
+    are never persisted.
    masked in each output line before it goes anywhere.
    """
    from steward.models.ansible import AnsibleRunStatus
    secrets = run_secrets(app.config.get("ANSIBLE", {}), connection, secret_vars, redact)
    _run_lines[run_id] = []
    db_output = ""
    truncated = False
@@ -474,7 +432,7 @@ async def start_run(
            assert proc.stdout is not None
            async for raw_line in proc.stdout:
-                line = scrub(raw_line.decode(errors="replace").rstrip("\n\r"), secrets)
+                line = raw_line.decode(errors="replace").rstrip("\n\r")
                _broadcast(run_id, line)
                if logf is not None:
                    logf.write(line + "\n")
@@ -520,7 +478,7 @@ async def start_run(
        final_status = "cancelled" if run_id in _cancelled else "failed"
        # Surface WHY in the run output instead of an empty "failed" (e.g. ENOSPC
        # when the temp dir can't be created — the run never reaches Ansible).
-        msg = scrub(f"[run error] {type(exc).__name__}: {exc}", secrets)
+        msg = f"[run error] {type(exc).__name__}: {exc}"
        _broadcast(run_id, msg)
        if len(failures) < _FAILURE_CAP:
            failures.append(msg[:500])
@@ -1,86 +0,0 @@
 """What a playbook run may be pointed at, and which variables only an admin
 may set.
 Ansible runs inside the Steward container, so a run is code on the Steward
 server unless it is held to playbooks its source lists and to variables that
 only shape what happens on the target. An operator may run playbooks and set
 their variables. Three things would let one run commands on Steward itself,
 and are refused for anyone but an admin:
 - a playbook or inventory path outside what the source lists;
 - an `ansible_*` variable that changes how or where Ansible connects
  (`ansible_connection: local`, `ansible_ssh_common_args` with a
  ProxyCommand, an interpreter or executable path);
 - a Jinja expression in any value (`{{ lookup('pipe', …) }}` is evaluated
  on the controller when the value is used).
 Playbook files themselves are admin-only to write (ansible/routes.py).
 """
 from __future__ import annotations
 from steward.ansible import sources as src_module
 from steward.models.users import UserRole
 # The ansible_* variables an operator may set: which address, port and user
 # to reach a target as, and whether to become another user there.
 OPERATOR_ANSIBLE_VARS = frozenset({
    "ansible_host", "ansible_port", "ansible_user", "ansible_password",
    "ansible_become", "ansible_become_user", "ansible_become_password",
 })
 _JINJA = ("{{", "{%", "{#")
 def check_playbook(source: dict, playbook_path: str) -> str | None:
    """None when the source lists this playbook, else why it is refused."""
    if playbook_path not in src_module.discover_playbooks(source["path"]):
        return f"Source {source['name']!r} has no playbook {playbook_path!r}"
    return None
 def check_inventory_scope(source: dict, scope: str) -> str | None:
    """None for a Steward scope or an inventory file the source has, else why not."""
    if scope.startswith("steward:"):
        return None
    if scope.startswith("repo:"):
        parts = scope.split(":", 2)
        path = parts[2] if len(parts) == 3 else ""
        if path in src_module.discover_inventories(source["path"]):
            return None
        return f"Source {source['name']!r} has no inventory file {path!r}"
    return "Invalid inventory_scope"
 def _has_jinja(value) -> bool:
    if isinstance(value, str):
        return any(mark in value for mark in _JINJA)
    if isinstance(value, dict):
        return any(_has_jinja(k) or _has_jinja(v) for k, v in value.items())
    if isinstance(value, (list, tuple)):
        return any(_has_jinja(v) for v in value)
    return False
 def check_vars_for_role(variables: dict | None, role: UserRole | None) -> str | None:
    """None when `role` may set these variables, else a message naming what
    only an admin may set. role=None is treated as the least privileged."""
    if role == UserRole.admin or not variables:
        return None
    names = sorted(n for n in variables
                   if str(n).startswith("ansible_") and n not in OPERATOR_ANSIBLE_VARS)
    if names:
        return (f"Only an admin can set {', '.join(names)}: these change how or where "
                "Ansible connects, and can run commands on the Steward server itself.")
    templated = sorted(str(k) for k, v in variables.items() if _has_jinja(k) or _has_jinja(v))
    if templated:
        return (f"Only an admin can set a value containing {{{{ }}}}, {{% %}} or {{# #}} "
                f"({', '.join(templated)}): Ansible evaluates it on the Steward server.")
    return None
 def changed_vars(new: dict | None, old: dict | None) -> dict:
    """The entries of `new` that are added or differ from `old`. An edit is
    held to the editor's role only for what it changes, so an operator can
    still edit a group whose connection vars an admin set."""
    old = old or {}
    return {k: v for k, v in (new or {}).items() if k not in old or old[k] != v}
@@ -106,39 +106,3 @@ async def fetch_scope_targets(db, scope: str) -> list:
    result = await db.execute(stmt)
    return list(result.scalars().all())
 def describe_scope(scope: str, targets: dict[str, str], groups: dict[str, str]) -> str:
    """A run's inventory scope in words: "target web-1", "group Docker Swarm",
    "all inventory targets", "<source>/<inventory file>". `targets` and
    `groups` map id → name; a scope naming a deleted one says so."""
    scope = scope or ""
    if scope == "steward:all":
        return "all inventory targets"
    for prefix, kind, names in (("steward:target:", "target", targets),
                                ("steward:group:", "group", groups)):
        if scope.startswith(prefix):
            ident = scope[len(prefix):]
            name = names.get(ident)
            return f"{kind} {name}" if name else f"{kind} {ident[:8]} (deleted)"
    if scope.startswith("repo:"):
        source, _, inv = scope[len("repo:"):].rpartition(":")
        return f"{source}/{inv}" if source else inv
    return scope
 async def scope_labels(db, scopes) -> dict[str, str]:
    """{scope: describe_scope(scope)} for the given scopes, in two queries."""
    from sqlalchemy import select
    from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget
    scopes = {s for s in scopes if s}
    t_ids = {s[len("steward:target:"):] for s in scopes if s.startswith("steward:target:")}
    g_ids = {s[len("steward:group:"):] for s in scopes if s.startswith("steward:group:")}
    targets = dict((await db.execute(
        select(AnsibleTarget.id, AnsibleTarget.name).where(AnsibleTarget.id.in_(t_ids))
    )).all()) if t_ids else {}
    groups = dict((await db.execute(
        select(AnsibleGroup.id, AnsibleGroup.name).where(AnsibleGroup.id.in_(g_ids))
    )).all()) if g_ids else {}
    return {s: describe_scope(s, targets, groups) for s in scopes}
@@ -3,13 +3,11 @@ from __future__ import annotations
 import uuid
 import yaml
-from quart import Blueprint, current_app, redirect, render_template, request, session, url_for
+from quart import Blueprint, current_app, redirect, render_template, request, url_for
 from sqlalchemy import select
 from sqlalchemy.orm import selectinload
 from steward.ansible import guards
 from steward.auth.middleware import require_role
 from steward.core.audit import audit_request as _audit
 from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget
 from steward.models.users import UserRole
@@ -32,12 +30,6 @@ def _parse_ansible_vars(raw: str) -> dict:
    return parsed
 def _vars_refusal(new: dict, old: dict | None = None) -> str | None:
    """Why this editor may not make this vars change, or None (ansible/guards.py)."""
    return guards.check_vars_for_role(guards.changed_vars(new, old),
                                      UserRole(session.get("user_role", "viewer")))
 # ---------------------------------------------------------------------------
 # Groups
 # ---------------------------------------------------------------------------
@@ -67,16 +59,10 @@ async def groups_create():
    except ValueError as exc:
        return str(exc), 400
    err = _vars_refusal(ansible_vars)
    if err:
        return err, 400
    group = AnsibleGroup(id=str(uuid.uuid4()), name=name, ansible_vars=ansible_vars)
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            db.add(group)
    await _audit("ansible.group.created", "ansible_group", group.id,
                 {"name": name, "vars": sorted(ansible_vars)})
    return redirect(url_for("ansible_inventory.groups_list"))
@@ -125,9 +111,6 @@ async def group_update(group_id: str):
            group = result.scalar_one_or_none()
            if group is None:
                return "Group not found", 404
            err = _vars_refusal(ansible_vars, group.ansible_vars)
            if err:
                return err, 400
            name = form.get("name", group.name).strip()
            if name:
@@ -140,10 +123,7 @@ async def group_update(group_id: str):
            ).scalars().all()
            target_map = {t.id: t for t in all_targets}
            group.targets = [target_map[tid] for tid in new_ids if tid in target_map]
            detail = {"name": group.name, "vars": sorted(ansible_vars),
                      "targets": sorted(t.name for t in group.targets)}
    await _audit("ansible.group.updated", "ansible_group", group_id, detail)
    return redirect(url_for("ansible_inventory.group_detail", group_id=group_id))
@@ -156,11 +136,8 @@ async def group_delete(group_id: str):
                select(AnsibleGroup).where(AnsibleGroup.id == group_id)
            )
            group = result.scalar_one_or_none()
-            if group is None:
+            if group is not None:
-                return redirect(url_for("ansible_inventory.groups_list"))
+                await db.delete(group)
            name = group.name
            await db.delete(group)
    await _audit("ansible.group.deleted", "ansible_group", group_id, {"name": name})
    return redirect(url_for("ansible_inventory.groups_list"))
@@ -194,18 +171,12 @@ async def targets_create():
    except ValueError as exc:
        return str(exc), 400
    err = _vars_refusal(ansible_vars)
    if err:
        return err, 400
    target = AnsibleTarget(
        id=str(uuid.uuid4()), name=name, address=address, ansible_vars=ansible_vars
    )
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            db.add(target)
    await _audit("ansible.target.created", "ansible_target", target.id,
                 {"name": name, "address": address, "vars": sorted(ansible_vars)})
    return redirect(url_for("ansible_inventory.targets_list"))
@@ -254,9 +225,6 @@ async def target_update(target_id: str):
            target = result.scalar_one_or_none()
            if target is None:
                return "Target not found", 404
            err = _vars_refusal(ansible_vars, target.ansible_vars)
            if err:
                return err, 400
            name = form.get("name", target.name).strip()
            address = form.get("address", target.address).strip()
@@ -270,10 +238,7 @@ async def target_update(target_id: str):
            all_groups_result = await db.execute(select(AnsibleGroup))
            group_map = {g.id: g for g in all_groups_result.scalars().all()}
            target.groups = [group_map[gid] for gid in new_group_ids if gid in group_map]
            detail = {"name": target.name, "address": target.address, "vars": sorted(ansible_vars),
                      "groups": sorted(g.name for g in target.groups)}
    await _audit("ansible.target.updated", "ansible_target", target_id, detail)
    return redirect(url_for("ansible_inventory.target_detail", target_id=target_id))
@@ -286,9 +251,6 @@ async def target_delete(target_id: str):
                select(AnsibleTarget).where(AnsibleTarget.id == target_id)
            )
            target = result.scalar_one_or_none()
-            if target is None:
+            if target is not None:
-                return redirect(url_for("ansible_inventory.targets_list"))
+                await db.delete(target)
            name = target.name
            await db.delete(target)
    await _audit("ansible.target.deleted", "ansible_target", target_id, {"name": name})
    return redirect(url_for("ansible_inventory.targets_list"))
@@ -8,10 +8,8 @@ from quart import (
 )
 from sqlalchemy import select
-from steward.ansible import executor, guards, sources as src_module
+from steward.ansible import executor, sources as src_module
 from steward.ansible.inventory_gen import scope_labels
 from steward.auth.middleware import require_role
 from steward.core.audit import audit_request as _audit
 from steward.models.ansible import AnsibleRun, AnsibleRunStatus
 from steward.models.ansible_schedule import AnsibleSchedule
 from steward.models.users import User, UserRole
@@ -106,30 +104,6 @@ async def view_playbook(source_name: str, playbook_path: str):
    )
 def _caller_role() -> UserRole:
    return UserRole(session.get("user_role", "viewer"))
 def _run_vars(params: dict | None, secret_vars: dict | None = None) -> dict:
    return {**((params or {}).get("extra_vars_map") or {}), **(secret_vars or {})}
 def check_run_form(form, source: dict | None, playbook_path: str,
                   params: dict | None, secret_vars: dict | None) -> str | None:
    """The server-side gates a web run must pass (the form's own checks are
    only a convenience): who may set these variables, and the playbook's
    steward:confirm flag, which a real (non-check) run must have ticked."""
    err = guards.check_vars_for_role(_run_vars(params, secret_vars), _caller_role())
    if err:
        return err
    content = src_module.read_playbook(source["path"], playbook_path) if source else None
    meta = src_module.discover_playbook_meta(content or "")
    if meta["confirm"] and not (params or {}).get("check") and "confirmed" not in form:
        return (f"{playbook_path} is flagged destructive by its author (steward:confirm). "
                "Tick the confirmation to run it, or run it in check mode.")
    return None
 def _parse_run_params(form) -> tuple[dict | None, dict | None, str | None]:
    """Build executor params + the unpersisted secret-vars dict from form fields.
@@ -260,10 +234,6 @@ async def create_run():
        return "playbook_path is required", 400
    params_or_none, secret_vars, err = _parse_run_params(form)
    if err:
        return err, 400
    source = next((s for s in _get_sources() if s["name"] == source_name), None)
    err = check_run_form(form, source, playbook_path, params_or_none, secret_vars)
    if err:
        return err, 400
@@ -280,10 +250,6 @@ async def create_run():
        return err, 404
    if err:
        return err, 400
    await _audit("ansible.run.started", "ansible_run", run.id, {
        "source": source_name, "playbook": playbook_path, "scope": inventory_scope,
        "check": bool((params_or_none or {}).get("check")),
        "vars": sorted(_run_vars(params_or_none, secret_vars))})
    return await render_template(
        "ansible/run_started.html",
@@ -340,10 +306,8 @@ async def run_detail(run_id: str):
            res = await db.execute(
                select(User.username).where(User.id == run.triggered_by))
            triggered_label = res.scalar_one_or_none() or "(deleted user)"
        scope_label = (await scope_labels(db, [run.inventory_scope])).get(run.inventory_scope)
    return await render_template(
-        "ansible/run_detail.html", run=run, triggered_label=triggered_label,
+        "ansible/run_detail.html", run=run, triggered_label=triggered_label)
        scope_label=scope_label)
@ansible_bp.post("/runs/<run_id>/cancel")
@@ -356,8 +320,6 @@ async def cancel_run(run_id: str):
        return "Not found", 404
    if run.status in (AnsibleRunStatus.running, AnsibleRunStatus.queued):
        executor.cancel_run(run_id)
        await _audit("ansible.run.cancelled", "ansible_run", run_id,
                     {"playbook": run.playbook_path})
    return redirect(url_for("ansible.run_detail", run_id=run_id))
@@ -406,7 +368,6 @@ async def schedules():
            r.id: r for r in (await db.execute(
                select(AnsibleRun).where(AnsibleRun.id.in_(run_ids)))).scalars().all()
        } if run_ids else {}
        labels = await scope_labels(db, [s.inventory_scope for s in scheds])
    source_data = [
        {"name": s["name"], "playbooks": src_module.discover_playbooks(s["path"])}
@@ -419,7 +380,6 @@ async def schedules():
        rows.append({
            "s": s,
            "last_run": runs.get(s.last_run_id) if s.last_run_id else None,
            "scope_label": labels.get(s.inventory_scope, s.inventory_scope),
            "next_run": (s.last_run_at + timedelta(seconds=s.interval_seconds)) if s.last_run_at else None,
        })
        if editing_id and s.id == editing_id:
@@ -476,15 +436,6 @@ def _schedule_form_fields(form) -> tuple[dict | None, str | None]:
    }, None
 def _schedule_detail(fields: dict) -> dict:
    """What a schedule audit row records: what runs, where, how often — and
    which variables, by name only."""
    return {"name": fields["name"], "source": fields["source_name"],
            "playbook": fields["playbook_path"], "scope": fields["inventory_scope"],
            "interval_seconds": fields["interval_seconds"],
            "vars": sorted(_run_vars(fields["params"]))}
@ansible_bp.post("/schedules")
@require_role(UserRole.operator)
 async def create_schedule():
@@ -492,14 +443,9 @@ async def create_schedule():
    fields, err = _schedule_form_fields(form)
    if err:
        return err, 400
    err = guards.check_vars_for_role(_run_vars(fields["params"]), _caller_role())
    if err:
        return err, 400
    sched = AnsibleSchedule(enabled=True, **fields)
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
-            db.add(sched)
+            db.add(AnsibleSchedule(enabled=True, **fields))
    await _audit("ansible.schedule.created", "ansible_schedule", sched.id, _schedule_detail(fields))
    return redirect(url_for("ansible.schedules"))
@@ -515,14 +461,8 @@ async def update_schedule(schedule_id: str):
            sched = await db.get(AnsibleSchedule, schedule_id)
            if sched is None:
                return "Not found", 404
            err = guards.check_vars_for_role(
                guards.changed_vars(_run_vars(fields["params"]), _run_vars(sched.params)),
                _caller_role())
            if err:
                return err, 400
            for k, v in fields.items():
                setattr(sched, k, v)
    await _audit("ansible.schedule.updated", "ansible_schedule", schedule_id, _schedule_detail(fields))
    return redirect(url_for("ansible.schedules"))
@@ -535,9 +475,6 @@ async def toggle_schedule(schedule_id: str):
            if sched is None:
                return "Not found", 404
            sched.enabled = not sched.enabled
            enabled, name = sched.enabled, sched.name
    await _audit("ansible.schedule.enabled" if enabled else "ansible.schedule.disabled",
                 "ansible_schedule", schedule_id, {"name": name})
    return redirect(url_for("ansible.schedules"))
@@ -547,11 +484,8 @@ async def delete_schedule(schedule_id: str):
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            sched = await db.get(AnsibleSchedule, schedule_id)
-            if sched is None:
+            if sched is not None:
-                return redirect(url_for("ansible.schedules"))
+                await db.delete(sched)
            name = sched.name
            await db.delete(sched)
    await _audit("ansible.schedule.deleted", "ansible_schedule", schedule_id, {"name": name})
    return redirect(url_for("ansible.schedules"))
@@ -580,8 +514,6 @@ async def run_schedule_now(schedule_id: str):
                fresh.last_run_at = datetime.now(timezone.utc)
                fresh.last_run_id = run.id if run else None
                fresh.last_error = err
    await _audit("ansible.schedule.run_now", "ansible_schedule", schedule_id,
                 {"playbook": playbook_path, "run_id": run.id if run else None, "error": err})
    if run:
        return redirect(url_for("ansible.run_detail", run_id=run.id))
    return redirect(url_for("ansible.schedules"))
@@ -658,9 +590,6 @@ async def save_playbook():
            content=content, error=err,
        ), 400
    await _audit("ansible.playbook.updated" if editing else "ansible.playbook.created",
                 "ansible_playbook", f"{source_name}:{playbook_path}",
                 {"source": source_name, "playbook": playbook_path, "bytes": len(content)})
    return redirect(url_for(
        "ansible.view_playbook", source_name=source_name, playbook_path=playbook_path))
@@ -672,12 +601,9 @@ async def delete_playbook():
    source = _editable_source((form.get("source_name", "") or "").strip())
    if source is None:
        return "Source is not editable", 400
-    playbook_path = (form.get("playbook_path", "") or "").strip()
+    ok, err = src_module.delete_playbook(source["path"], (form.get("playbook_path", "") or "").strip())
    ok, err = src_module.delete_playbook(source["path"], playbook_path)
    if not ok:
        return err, 400
    await _audit("ansible.playbook.deleted", "ansible_playbook", f"{source['name']}:{playbook_path}",
                 {"source": source["name"], "playbook": playbook_path})
    return redirect(url_for("ansible.browse"))
@@ -10,7 +10,7 @@ import asyncio
 import uuid
 from datetime import datetime, timezone
-from steward.ansible import executor, guards, sources as src_module
+from steward.ansible import executor, sources as src_module
 from steward.ansible.inventory_gen import (
    fetch_scope_targets, generate_inventory, inventory_to_yaml,
 )
@@ -28,7 +28,6 @@ async def trigger_run(
    inventory_content: str | None = None,
    connection: dict | None = None,
    secret_vars: dict | None = None,
    redact: list[str] | None = None,
 ):
    """Resolve inventory for the scope, persist an AnsibleRun, and launch it.
@@ -40,8 +39,7 @@ async def trigger_run(
    first-contact provisioning — passed to the executor but deliberately NOT
    stored on the AnsibleRun row (params), so the password never lands in the DB.
    secret_vars (sensitive run-time playbook variables) are likewise passed
-    through to the executor but never persisted. redact names further values
+    through to the executor but never persisted.
    to mask in the run's output (a bespoke inventory's per-host tokens).
    Returns (run, source, error): on success error is None; on failure run is
    None and error is a short human-readable reason.
    """
@@ -49,13 +47,6 @@ async def trigger_run(
    source = next((s for s in sources if s["name"] == source_name), None)
    if source is None:
        return None, None, "Source not found"
    # Every run path (web, MCP, alerts, schedules, plugins) arrives here, so
    # this is where a run is held to what its source lists (ansible/guards.py).
    err = guards.check_playbook(source, playbook_path)
    if err is None and inventory_content is None:
        err = guards.check_inventory_scope(source, inventory_scope)
    if err:
        return None, source, err
    inventory_path: str | None = None
    if inventory_content is not None:
@@ -64,8 +55,11 @@ async def trigger_run(
        async with app.db_sessionmaker() as db:
            targets = await fetch_scope_targets(db, inventory_scope)
        inventory_content = inventory_to_yaml(generate_inventory(targets))
    elif inventory_scope.startswith("repo:"):
        parts = inventory_scope.split(":", 2)
        inventory_path = parts[2] if len(parts) == 3 else ""
    else:
-        inventory_path = inventory_scope.split(":", 2)[2]
+        return None, source, "Invalid inventory_scope"
    run_id = str(uuid.uuid4())
    run = AnsibleRun(
@@ -87,7 +81,7 @@ async def trigger_run(
        executor.start_run(
            app, run_id, playbook_path, inventory_path or "",
            source["path"], params or None, inventory_content,
-            connection=connection, secret_vars=secret_vars, redact=redact,
+            connection=connection, secret_vars=secret_vars,
        )
    )
    task.add_done_callback(
@@ -108,13 +108,6 @@ def is_editable_source(source: dict) -> bool:
    return source.get("type") == "local" and source.get("name") != BUILTIN_SOURCE_NAME
 def _open_token(src: dict) -> str:
    """The source's git token in the clear (it is stored sealed)."""
    from steward.core.crypto import decrypt_secret
    from steward.core.settings import source_token_key
    return decrypt_secret(src.get("http_token") or "", context=source_token_key(src.get("name", "?")))
 def get_sources(ansible_cfg: dict) -> list[dict]:
    """Return resolved source list from ansible config section.
@@ -156,7 +149,7 @@ def get_sources(ansible_cfg: dict) -> list[dict]:
            "url": src.get("url"),
            "branch": src.get("branch", "main"),
            "pull_interval_seconds": int(src.get("pull_interval_seconds", 3600)),
-            "http_token": _open_token(src) if src_type == "git" else "",
+            "http_token": src.get("http_token", "") if src_type == "git" else "",
        })
    return result
@@ -330,25 +323,11 @@ def validate_playbook_yaml(content: str) -> tuple[bool, str | None]:
    return True, None
 # The askpass script holds no secret: git runs it with the token in its
 # environment, and printf prints it as data, so no character in a token can
 # become shell syntax.
 ASKPASS_TOKEN_ENV = "STEWARD_GIT_TOKEN"
 ASKPASS_SCRIPT = (
    "#!/bin/sh\n"
    'case "$1" in\n'
    "  Username*) printf '%s\\n' oauth2 ;;\n"
    f"  Password*) printf '%s\\n' \"${ASKPASS_TOKEN_ENV}\" ;;\n"
    "esac\n"
 )
 async def git_pull(source: dict) -> None:
    """Clone the git repo if absent; pull if already present.
-    When source['http_token'] is set, git asks a temporary GIT_ASKPASS script,
+    When source['http_token'] is set, injects credentials via a temporary
-    which prints the token from the child's environment, so it never touches
+    GIT_ASKPASS script so they never touch .git/config or process args.
    .git/config, process args or a file.
    """
    path = Path(source["path"])
    token = source.get("http_token", "")
@@ -358,11 +337,18 @@ async def git_pull(source: dict) -> None:
    if token:
        fd, askpass_path = tempfile.mkstemp(prefix="steward-askpass-", suffix=".sh")
        try:
-            os.write(fd, ASKPASS_SCRIPT.encode())
+            script = (
                "#!/bin/sh\n"
                'case "$1" in\n'
                '  Username*) echo "oauth2" ;;\n'
                f'  Password*) echo "{token}" ;;\n'
                "esac\n"
            )
            os.write(fd, script.encode())
        finally:
            os.close(fd)
        os.chmod(askpass_path, stat.S_IRWXU)
-        env = {**os.environ, "GIT_ASKPASS": askpass_path, ASKPASS_TOKEN_ENV: token}
+        env = {**os.environ, "GIT_ASKPASS": askpass_path}
    try:
        if not (path / ".git").exists():
@@ -3,12 +3,9 @@ from __future__ import annotations
 import asyncio
 import logging
 from pathlib import Path
-from quart import Quart, render_template, request
+from quart import Quart, render_template
 from .config import load_bootstrap
-from .database import (
+from .database import init_db
    init_db, ensure_database_reachable, DatabaseUnavailable,
    DB_CONNECT_TIMEOUT_SECONDS,
 )
 logger = logging.getLogger(__name__)
@@ -26,7 +23,6 @@ def create_app(
        bootstrap = {
            "database_url": "postgresql+asyncpg://test/test",
            "secret_key": "test-secret-key",
            "db_connect_timeout": DB_CONNECT_TIMEOUT_SECONDS,
            "plugin_dirs": ["plugins"],
            "plugin_install_dir": "plugins",
        }
@@ -37,9 +33,6 @@ def create_app(
        PLUGIN_DIRS=bootstrap["plugin_dirs"],
        PLUGIN_INSTALL_DIR=bootstrap["plugin_install_dir"],
        TESTING=testing,
        # Lax keeps the session cookie off cross-site POSTs and iframes; the
        # CSRF token below is the check that does not depend on the browser.
        SESSION_COOKIE_SAMESITE="Lax",
    )
    # ── 2. Init DB engine ──────────────────────────────────────────────────────
@@ -49,19 +42,6 @@ def create_app(
        from unittest.mock import MagicMock
        app.db_sessionmaker = MagicMock()
    # ── 2b. Block until the database is actually reachable ────────────────────
    # Everything from here down touches the DB synchronously (migrations,
    # secret re-encryption, settings load). Gate all of it behind one readiness
    # check so a DB that is merely slow to come up — WAL recovery after an
    # unclean shutdown, or Docker DNS not yet serving the `db` name after a host
    # reboot — is waited out instead of crashing the container on a traceback.
    if not testing:
        from .database import wait_for_database
        wait_for_database(
            app.config["DATABASE_URL"],
            timeout_seconds=bootstrap["db_connect_timeout"],
        )
    # ── 3. Core migrations only (creates app_settings table) ──────────────────
    if not testing:
        from .core.migration_runner import run_core_migrations
@@ -90,14 +70,13 @@ def create_app(
        from .core.settings import (
            load_settings_sync, to_smtp_cfg, to_webhook_cfg,
            to_ansible_cfg, to_plugins_cfg, to_oidc_cfg, to_ldap_cfg,
-            to_thresholds_cfg, to_access_cfg,
+            to_thresholds_cfg,
        )
        _settings = load_settings_sync(app.config["DATABASE_URL"])
        app.config.update(to_access_cfg(_settings))
        app.config.update(
            SESSION_LIFETIME_HOURS=_settings.get("session.lifetime_hours", 8),
            DATA_RETENTION_DAYS=_settings.get("data.retention_days", 90),
            MONITORS_POLL_INTERVAL=_settings.get("monitors.poll_interval_seconds", 60),
            PUBLIC_BASE_URL=_settings.get("general.public_base_url", ""),
            SMTP=to_smtp_cfg(_settings),
            WEBHOOK=to_webhook_cfg(_settings),
            ANSIBLE=to_ansible_cfg(_settings),
@@ -107,12 +86,11 @@ def create_app(
            THRESHOLDS=to_thresholds_cfg(_settings),
        )
    else:
-        from .core.settings import to_access_cfg, to_thresholds_cfg
+        from .core.settings import to_thresholds_cfg
        app.config.update(to_access_cfg({}))
        app.config.update(
            SESSION_LIFETIME_HOURS=8,
            DATA_RETENTION_DAYS=90,
            MONITORS_POLL_INTERVAL=60,
            PUBLIC_BASE_URL="",
            SMTP={},
            WEBHOOK={},
            ANSIBLE={},
@@ -130,48 +108,6 @@ def create_app(
        lambda value, kind: _ts_for(value, kind, app.config.get("THRESHOLDS", {}))
    )
    # One way to write a size, a rate, a span and a moment, and to grade a
    # reading ('ok' / 'warn' / 'crit' / 'unknown') against the same cutoffs.
    from .core import formatting as _fmt
    app.jinja_env.filters["bytes"] = _fmt.human_bytes
    app.jinja_env.filters["rate"] = _fmt.human_rate
    app.jinja_env.filters["duration"] = _fmt.human_duration
    app.jinja_env.filters["ago"] = _fmt.ago
    app.jinja_env.globals["status_class"] = (
        lambda value, kind: _fmt.status_class(value, kind, app.config.get("THRESHOLDS", {}))
    )
    # Forged-request protection: every POST form renders csrf_field(), HTMX and
    # fetch() send csrf_token() as X-CSRF-Token, and one check refuses any
    # unsafe request without it (steward/auth/csrf.py).
    from .auth import csrf as _csrf
    app.jinja_env.globals["csrf_token"] = _csrf.csrf_token
    app.jinja_env.globals["csrf_field"] = _csrf.csrf_field
    app.before_request(_csrf.check_request)
    # The session cookie is Secure, and HSTS is sent, only on requests that
    # arrived over HTTPS — directly or as a trusted proxy reports it
    # (steward/core/proxy.py). Plain-HTTP and LAN use keep working.
    from quart import has_request_context
    from quart.sessions import SecureCookieSessionInterface
    from .core.proxy import is_https
    class _SessionInterface(SecureCookieSessionInterface):
        def get_cookie_secure(self, app):
            return bool(app.config.get("SESSION_COOKIE_SECURE")) or (
                has_request_context() and is_https())
    app.session_interface = _SessionInterface()
    @app.after_request
    async def _hsts(response):
        if is_https():
            response.headers.setdefault("Strict-Transport-Security", "max-age=31536000")
        return response
    # Default-deny: a view with neither require_role nor public_route is refused.
    from .auth.middleware import check_view_is_marked, public_route
    app.before_request(check_view_is_marked)
    # ── 5. Plugin migrations ───────────────────────────────────────────────────
    # Step 3 already applied all plugin migrations via discover_all_plugin_migration_dirs,
    # so this is a no-op on normal startup. We still run it with all discovered dirs so
@@ -197,7 +133,6 @@ def create_app(
    from .ansible.inventory_routes import inventory_bp
    from .settings.routes import settings_bp
    from .audit.routes import audit_bp
    from .ai_access.routes import ai_access_bp
    app.register_blueprint(auth_bp)
    app.register_blueprint(dashboard_bp)
@@ -209,7 +144,6 @@ def create_app(
    app.register_blueprint(inventory_bp)
    app.register_blueprint(settings_bp)
    app.register_blueprint(audit_bp)
    app.register_blueprint(ai_access_bp)
    # Register the unified Monitor status source for the Status page. Plugins
    # may register additional sources from setup(). Idempotent.
@@ -267,50 +201,36 @@ def create_app(
    # ── 11. Share-token middleware ─────────────────────────────────────────────
    @app.before_request
    async def _validate_share_token():
-        """A share link's token (?s=) opens, read-only, exactly the widget
+        """If ?s=<token> is present, validate it and set g.share_viewer."""
        fragments its own dashboard loads (steward/dashboard/share.py)."""
        from quart import request, g
        raw = request.args.get("s")
-        if not raw or request.method != "GET":
+        if not raw:
            return
        try:
-            from .dashboard.share import share_grants
+            import hashlib
            from datetime import datetime, timezone
            from sqlalchemy import select
            from .models.dashboard import DashboardShareToken
            token_hash = hashlib.sha256(raw.encode()).hexdigest()
            async with app.db_sessionmaker() as db:
-                if await share_grants(db, raw, request.path, request.args):
+                result = await db.execute(
                    select(DashboardShareToken)
                    .where(DashboardShareToken.token_hash == token_hash)
                )
                share = result.scalar_one_or_none()
            if share:
                now = datetime.now(timezone.utc)
                if share.expires_at is None or share.expires_at > now:
                    g.share_viewer = True
                    g.share_dashboard_id = share.dashboard_id
        except Exception:
-            # A failed check grants nothing; the request falls through to login.
+            pass  # never block a request due to token validation failure
            logging.getLogger(__name__).exception("Share token check failed")
    # ── 11. Health check ───────────────────────────────────────────────────────
    @app.get("/health")
    @public_route("liveness probe for the container")
    async def health():
        return {"status": "ok"}
    # ── 11b. Database availability gate ────────────────────────────────────────
    # Every page in Steward reads the database, so a database that has gone away
    # under a running app turns each request into an opaque 500 (this is how a
    # login attempt surfaced a bare gaierror). Acquire a connection up front,
    # retrying briefly to ride out a restart, and answer honestly if it stays
    # down rather than failing deep inside a handler with a generic error.
    if not testing:
        @app.before_request
        async def _database_gate():
            # /health is a liveness probe for the container itself — it must
            # stay answerable while the database is down, or a restart loop
            # gets triggered by a dependency outage. Static files need no DB.
            if request.endpoint in ("health", "static"):
                return None
            try:
                await ensure_database_reachable(app._db_engine)
            except DatabaseUnavailable as exc:
                logger.error("Database unreachable while serving %s: %s",
                             request.path, exc)
                return await render_template(
                    "errors/database_unavailable.html"), 503
            return None
    # ── 12. Error handlers ─────────────────────────────────────────────────────
    @app.errorhandler(404)
    async def not_found(_):
@@ -320,17 +240,6 @@ def create_app(
    async def server_error(_):
        return await render_template("errors/500.html"), 500
    # ── Session freshness: lifetime, deactivation and role changes apply per request
    if not testing:
        from .auth.middleware import refresh_session_user
        app.before_request(refresh_session_user)
    # ── MCP endpoint (/mcp) ───────────────────────────────────────────────────
    # Mounted after plugins load so their register_mcp_tools() hooks are seen.
    # It wraps app.asgi_app: /mcp never enters Quart's pipeline (see mcp/server.py).
    from .mcp import mount_mcp
    mount_mcp(app)
    # ── 11. Startup hook ───────────────────────────────────────────────────────
    @app.before_serving
    async def startup():
@@ -355,14 +264,6 @@ def _register_core_tasks(app: Quart) -> None:
        from .core.cleanup import run_cleanup as _cleanup
        await _cleanup(app)
    async def run_alert_reconcile():
        from .core.alerts import check_silence, notify_after_maintenance
        for step in (check_silence, notify_after_maintenance):
            try:
                await step(app)
            except Exception:
                app.logger.exception("alert_reconcile: %s failed", step.__name__)
    async def run_report_check():
        from .core.reports import check_and_send
        await check_and_send(app)
@@ -387,14 +288,6 @@ def _register_core_tasks(app: Quart) -> None:
            interval_seconds=60,
            run_on_startup=True,
        ),
        # Rules whose series has gone quiet get a visible no_data mark; alerts
        # a maintenance window muted notify once it ends.
        ScheduledTask(
            name="alert_reconcile",
            coro_factory=run_alert_reconcile,
            interval_seconds=60,
            run_on_startup=False,
        ),
        ScheduledTask(
            name="monitor_check",
            coro_factory=run_monitor_checks,
@@ -1,63 +0,0 @@
 """Forged-request (CSRF) protection for the browser side of Steward.
 A page on another site can make a logged-in browser POST to Steward, and the
 browser sends Steward's session cookie with it. Every state-changing request
 from a browser therefore has to carry a token that only Steward's own pages
 know: a random value kept in the session, written into each form by
 `csrf_field()` and sent by HTMX and fetch() as the `X-CSRF-Token` header (set
 once on <body> in base.html).
 One check in a before_request covers every route, so a new route cannot
 forget it. Callers that authenticate with a bearer token instead of the
 session cookie (the host agent's ingest; /mcp never reaches Quart) are exempt:
 a forged browser request cannot carry their token.
 """
 from __future__ import annotations
 import hmac
 import secrets
 from markupsafe import Markup, escape
 from quart import Response, request, session
 SESSION_KEY = "_csrf"
 FORM_FIELD = "csrf_token"
 HEADER = "X-CSRF-Token"
 UNSAFE_METHODS = ("POST", "PUT", "PATCH", "DELETE")
 # Bearer-authenticated endpoints: no session cookie, nothing to forge.
 EXEMPT_PATHS = frozenset({"/plugins/host_agent/ingest"})
 def csrf_token() -> str:
    """This session's token, created on first use."""
    token = session.get(SESSION_KEY)
    if not token:
        token = session[SESSION_KEY] = secrets.token_urlsafe(32)
    return token
 def csrf_field() -> Markup:
    """The hidden input every POST form carries."""
    return Markup(f'<input type="hidden" name="{FORM_FIELD}" value="{escape(csrf_token())}">')
 def rotate() -> None:
    """Drop the token, so a new one is issued — at login, where the session
    changes hands."""
    session.pop(SESSION_KEY, None)
 async def check_request() -> Response | None:
    """before_request hook: refuse an unsafe request without the session's token."""
    if request.method not in UNSAFE_METHODS or request.path in EXEMPT_PATHS:
        return None
    expected = session.get(SESSION_KEY)
    sent = request.headers.get(HEADER, "")
    if not sent:
        form = await request.form
        sent = form.get(FORM_FIELD, "")
    if expected and sent and hmac.compare_digest(expected, sent):
        return None
    return Response(
        "This form has expired or did not come from Steward. Reload the page and try again.",
        status=400, content_type="text/plain; charset=utf-8")
@@ -18,34 +18,6 @@ def _ldap_available() -> bool:
        return False
 def filter_value(value: str) -> str:
    """Escape a value for an LDAP search filter (RFC 4515), so a login name
    like `*)(uid=*` matches only itself."""
    out = []
    for ch in value:
        if ch in "\\*()\x00":
            out.append("\\%02x" % ord(ch))
        else:
            out.append(ch)
    return "".join(out)
 def transport(cfg: dict) -> tuple[bool, bool]:
    """(use_ssl, start_tls) for the configured server: port 636 is LDAPS,
    otherwise the TLS setting means StartTLS before any bind."""
    if int(cfg.get("port", 389)) == 636:
        return True, False
    return False, bool(cfg.get("tls", False))
 def _tls(ldap3, cfg: dict):
    """Certificate-checked TLS: against the CA the operator pasted in, else the
    system's CAs. A server whose certificate does not check is refused."""
    import ssl
    ca = (cfg.get("ca_cert") or "").strip()
    return ldap3.Tls(validate=ssl.CERT_REQUIRED, ca_certs_data=ca or None)
 def _ldap_authenticate_sync(cfg: dict, username: str, password: str) -> dict | None:
    """Synchronous LDAP auth. Returns user info dict or None on failure.
@@ -55,23 +27,22 @@ def _ldap_authenticate_sync(cfg: dict, username: str, password: str) -> dict | N
    host = cfg.get("host", "")
    port = int(cfg.get("port", 389))
    use_tls = cfg.get("tls", False)
    bind_dn = cfg.get("bind_dn", "")
    bind_password = cfg.get("bind_password", "")
    base_dn = cfg.get("base_dn", "")
-    user_filter = cfg.get("user_filter", "(uid={username})").replace("{username}", filter_value(username))
+    user_filter = cfg.get("user_filter", "(uid={username})").format(username=username)
    admin_group_dn = cfg.get("admin_group_dn", "")
    operator_group_dn = cfg.get("operator_group_dn", "")
    attr_username = cfg.get("attr_username", "uid")
    attr_email = cfg.get("attr_email", "mail")
-    use_ssl, start_tls = transport(cfg)
+    tls = ldap3.Tls() if use_tls else None
-    tls = _tls(ldap3, cfg) if (use_ssl or start_tls) else None
+    server = ldap3.Server(host, port=port, tls=tls, get_info=ldap3.ALL)
    server = ldap3.Server(host, port=port, use_ssl=use_ssl, tls=tls, get_info=ldap3.ALL)
    auto_bind = ldap3.AUTO_BIND_TLS_BEFORE_BIND if start_tls else ldap3.AUTO_BIND_NO_TLS
    # Service account bind to search for user DN
    try:
-        svc_conn = ldap3.Connection(server, user=bind_dn, password=bind_password, auto_bind=auto_bind)
+        svc_conn = ldap3.Connection(server, user=bind_dn, password=bind_password, auto_bind=True)
    except Exception as exc:
        logger.error("LDAP service bind failed: %s", exc)
        return None
@@ -98,7 +69,7 @@ def _ldap_authenticate_sync(cfg: dict, username: str, password: str) -> dict | N
    # Bind as the user to verify password
    try:
-        user_conn = ldap3.Connection(server, user=user_dn, password=password, auto_bind=auto_bind)
+        user_conn = ldap3.Connection(server, user=user_dn, password=password, auto_bind=True)
        user_conn.unbind()
    except Exception:
        logger.debug("LDAP password verify failed for %s", username)
@@ -112,7 +83,6 @@ def _ldap_authenticate_sync(cfg: dict, username: str, password: str) -> dict | N
        role = "operator"
    return {
        "subject": user_dn,
        "username": ldap_username,
        "email": ldap_email,
        "role": role,
@@ -120,12 +90,7 @@ def _ldap_authenticate_sync(cfg: dict, username: str, password: str) -> dict | N
 async def ldap_authenticate(cfg: dict, username: str, password: str) -> dict | None:
-    """Async wrapper around synchronous LDAP auth. Returns user info or None.
+    """Async wrapper around synchronous LDAP auth. Returns user info or None."""
    An empty password is refused here: LDAP treats a simple bind with a DN
    and no password as an anonymous bind, which succeeds."""
    if not username or not password:
        return None
    if not _ldap_available():
        logger.warning("ldap3 not installed — LDAP auth unavailable")
        return None
@@ -1,6 +1,5 @@
 from __future__ import annotations
 import functools
 import time
 from urllib.parse import quote, urlsplit
 from quart import session, redirect, url_for, abort, g, request, Response
 from steward.models.users import UserRole
@@ -78,74 +77,15 @@ def require_role(minimum_role: UserRole):
            if _ROLE_ORDER.index(user_role) < _ROLE_ORDER.index(minimum_role):
                abort(403)
            return await f(*args, **kwargs)
        wrapper.steward_role = minimum_role
        return wrapper
    return decorator
 def public_route(reason: str):
    """Mark a view that is deliberately reachable without a role — login,
    the health probe, a bearer-authenticated endpoint. Every other view must
    carry require_role; check_view_is_marked refuses one that has neither, so
    a route that forgets its decorator is closed rather than open."""
    def decorator(f):
        f.steward_public = reason
        return f
    return decorator
 async def check_view_is_marked() -> Response | None:
    """before_request hook: default-deny for views with neither require_role
    nor public_route. Static files and unmatched paths (404) pass through."""
    from quart import current_app
    endpoint = request.endpoint
    if endpoint is None or endpoint == "static" or endpoint.endswith(".static"):
        return None
    view = current_app.view_functions.get(endpoint)
    if view is None or hasattr(view, "steward_role") or hasattr(view, "steward_public"):
        return None
    current_app.logger.error("Refused %s: view %r has no require_role or public_route",
                             request.path, endpoint)
    abort(403)
 def login_user(user) -> None:
-    """Write user identity into the session, with a fresh CSRF token."""
+    """Write user identity into the session."""
    from steward.auth.csrf import rotate
    rotate()
    session.permanent = True
    session["user_id"] = user.id
    session["user_role"] = user.role.value
    session["username"] = user.username
    session["login_at"] = time.time()
 async def refresh_session_user() -> None:
    """before_request: a session speaks for the user as they are now.
    A login ends once Settings → General's session lifetime has passed since
    it began (not since the last request: the cookie is re-signed on every
    response, so the cookie's own age would let an active session live
    forever). A user who has been deleted or deactivated is logged out on
    their next request, and a role change applies at once.
    """
    user_id = session.get("user_id")
    if not user_id:
        return
    from quart import current_app
    from steward.models.users import User
    lifetime = current_app.config.get("PERMANENT_SESSION_LIFETIME")
    started = session.get("login_at") or 0
    if lifetime is not None and time.time() - started > lifetime.total_seconds():
        session.clear()
        return
    async with current_app.db_sessionmaker() as db:
        user = await db.get(User, user_id)
    if user is None or not user.is_active:
        session.clear()
        return
    if session.get("user_role") != user.role.value:
        session["user_role"] = user.role.value
 def logout_user() -> None:
@@ -95,25 +95,6 @@ def decode_id_token_payload(id_token: str) -> dict:
        return {}
 def id_token_refusal(id_token: str, nonce: str | None, userinfo_sub: str) -> str | None:
    """Why this sign-in's ID token does not belong to it, or None when it does.
    The token must carry the nonce this browser was sent off with, so a token
    from another sign-in cannot be replayed into this one, and name the same
    subject as the userinfo response (OIDC Core 5.3.2). The signature is not
    checked (see the module note)."""
    if not userinfo_sub:
        return "SSO returned no subject (sub) for this user."
    claims = decode_id_token_payload(id_token) if id_token else {}
    if not claims:
        return "SSO returned no ID token. Check that the scopes include openid."
    if not nonce or claims.get("nonce") != nonce:
        return "This SSO sign-in does not match the one Steward started. Try again."
    if claims.get("sub") != userinfo_sub:
        return "SSO returned an ID token for a different user. Try again."
    return None
 def map_role(groups: list[str], admin_group: str, operator_group: str) -> str:
    """Map IdP groups to a local role string."""
    if admin_group and admin_group in groups:
@@ -2,71 +2,37 @@ from __future__ import annotations
 import bcrypt
 from quart import Blueprint, render_template, request, redirect, url_for, session
 from sqlalchemy import select, func
-from steward.auth.middleware import login_user, logout_user, public_route, safe_next_url
+from steward.auth.middleware import login_user, logout_user, safe_next_url
 from steward.auth.throttle import login_throttle
 from steward.core.proxy import client_ip
 from steward.models.users import User, UserRole
 auth_bp = Blueprint("auth", __name__)
-_SOURCE_LABEL = {"ldap": "LDAP", "oidc": "SSO"}
+async def _provision_external_user(app, username: str, email: str, role_str: str):
    """Create or update a user from an external auth provider (OIDC/LDAP).
-
+    External users get a randomised unusable password so local login is blocked.
-async def _provision_external_user(app, source: str, subject: str, username: str,
+    Their role is updated on every login to reflect current IdP group membership.
                                   email: str, role_str: str) -> tuple[User | None, str | None]:
    """The Steward user for an identity a provider vouched for: (user, None),
    or (None, the reason to show at the login page).
    Found by (source, subject) only. A first sign-in creates the user with an
    unusable password; a username or email another account already holds is
    refused rather than merged, so a directory entry named `admin` cannot sign
    in as the local admin. The role follows the provider's groups on every
    sign-in; a user disabled in Steward stays out.
    """
    import secrets
-    from sqlalchemy.exc import IntegrityError
+    from steward.models.users import UserRole
    label = _SOURCE_LABEL[source]
    if not subject or len(subject) > 512 or not username or len(username) > 64:
        return None, f"{label} returned a username or id Steward cannot store (at most 64 characters)."
    new_role = UserRole(role_str)
    async with app.db_sessionmaker() as db:
-        try:
+        result = await db.execute(select(User).where(User.username == username))
-            async with db.begin():
+        user = result.scalar_one_or_none()
-                user = (await db.execute(select(User).where(
+        new_role = UserRole(role_str)
-                    User.auth_source == source, User.external_id == subject))).scalar_one_or_none()
+        async with db.begin():
-                if user is not None:
+            if user is None:
-                    if not user.is_active:
+                unusable_hash = bcrypt.hashpw(secrets.token_bytes(32), bcrypt.gensalt()).decode()
-                        return None, "This account is disabled in Steward."
+                user = User(username=username, email=email or f"{username}@external",
                            password_hash=unusable_hash, role=new_role)
                db.add(user)
            else:
                if user.role != new_role:
                    user.role = new_role
-                else:
+    # Re-fetch committed user
-                    email = email or f"{username}@{source}"
+    async with app.db_sessionmaker() as db:
-                    clash = (await db.execute(select(User.username).where(
+        result = await db.execute(select(User).where(User.username == username))
-                        (User.username == username) | (User.email == email)))).first()
+        return result.scalar_one()
                    if clash is not None:
                        return None, (
                            f"Steward already has an account named {clash[0]} with this username "
                            f"or email, and it does not sign in with {label}. An admin must rename "
                            f"or remove it before this {label} account can sign in.")
                    user = User(username=username, email=email, role=new_role,
                                password_hash=bcrypt.hashpw(secrets.token_bytes(32),
                                                            bcrypt.gensalt()).decode(),
                                auth_source=source, external_id=subject)
                    db.add(user)
        except IntegrityError:
            # Another sign-in created the same name between our check and insert.
            return None, f"Could not create the {label} account; try signing in again."
        db.expunge(user)
    return user, None
 async def _login_refused(message: str, next_url: str | None = None):
    """The login page with why a provider-vouched sign-in was still refused."""
    from quart import current_app
    return await render_template(
        "auth/login.html", error=message,
        oidc_enabled=current_app.config.get("OIDC", {}).get("enabled", False),
        next_url=safe_next_url(next_url)), 403
 async def get_user_count(app) -> int:
@@ -82,7 +48,6 @@ async def get_user_by_username(app, username: str) -> User | None:
@auth_bp.get("/login")
@public_route("the login page")
 async def login():
    from quart import current_app
    if await get_user_count(current_app) == 0:
@@ -94,7 +59,6 @@ async def login():
@auth_bp.post("/login")
@public_route("the login form")
 async def login_post():
    from quart import current_app
    from steward.core.audit import log_audit
@@ -104,26 +68,9 @@ async def login_post():
    password = password_str.encode()
    dest = safe_next_url(form.get("next")) or url_for("dashboard.index")
    # Too many recent failures from this address: refuse before checking anything.
    ip = client_ip()
    wait_s = login_throttle.retry_after(
        ip, current_app.config.get("LOGIN_MAX_FAILURES", 5),
        current_app.config.get("LOGIN_LOCKOUT_MINUTES", 15) * 60)
    if wait_s:
        oidc_cfg = current_app.config.get("OIDC", {})
        return await render_template(
            "auth/login.html",
            error=f"Too many failed logins from this address. Try again in "
                  f"{max(1, round(wait_s / 60))} minute{'s' if wait_s > 90 else ''}.",
            oidc_enabled=oidc_cfg.get("enabled", False),
            next_url=safe_next_url(form.get("next")),
        ), 429
    # Try local auth first
    user = await get_user_by_username(current_app, username)
-    if (user and user.auth_source == "local" and user.is_active
+    if user and user.is_active and bcrypt.checkpw(password, user.password_hash.encode()):
            and bcrypt.checkpw(password, user.password_hash.encode())):
        login_throttle.succeeded(ip)
        login_user(user)
        await log_audit(current_app, user.id, user.username, "auth.login",
                        detail={"method": "local", "role": user.role.value})
@@ -135,20 +82,14 @@ async def login_post():
        from steward.auth.ldap_auth import ldap_authenticate
        ldap_info = await ldap_authenticate(ldap_cfg, username, password_str)
        if ldap_info:
-            user, refusal = await _provision_external_user(
+            user = await _provision_external_user(
-                current_app, "ldap", ldap_info["subject"], ldap_info["username"],
+                current_app, ldap_info["username"], ldap_info["email"], ldap_info["role"]
-                ldap_info["email"], ldap_info["role"])
+            )
            if refusal:
                return await _login_refused(refusal, form.get("next"))
            login_throttle.succeeded(ip)
            login_user(user)
            await log_audit(current_app, user.id, user.username, "auth.login",
                            detail={"method": "ldap", "role": user.role.value})
            return redirect(dest)
    login_throttle.failed(ip)
    await log_audit(current_app, None, username or "(blank)", "auth.login_failed",
                    detail={"address": ip})
    oidc_cfg = current_app.config.get("OIDC", {})
    return await render_template(
        "auth/login.html",
@@ -159,7 +100,6 @@ async def login_post():
@auth_bp.get("/login/oidc")
@public_route("starts SSO login")
 async def login_oidc():
    from quart import current_app
    from steward.auth.oidc import get_discovery, build_authorize_url, generate_state
@@ -185,11 +125,9 @@ async def login_oidc():
@auth_bp.get("/login/oidc/callback")
@public_route("SSO login callback")
 async def login_oidc_callback():
    from quart import current_app
-    from steward.auth.oidc import (get_discovery, exchange_code, get_userinfo, map_role,
+    from steward.auth.oidc import get_discovery, exchange_code, get_userinfo, map_role
                                    id_token_refusal)
    from steward.core.audit import log_audit
    oidc_cfg = current_app.config.get("OIDC", {})
@@ -205,7 +143,6 @@ async def login_oidc_callback():
        return await render_template("auth/login.html", error="Invalid SSO state — try again.",
                                     oidc_enabled=True), 400
    nonce = session.pop("oidc_nonce", None)
    try:
        doc = await get_discovery(oidc_cfg["discovery_url"])
        redirect_uri = url_for("auth.login_oidc_callback", _external=True)
@@ -217,25 +154,23 @@ async def login_oidc_callback():
        return await render_template("auth/login.html", error=f"SSO login failed: {exc}",
                                     oidc_enabled=True), 502
    refusal = id_token_refusal(tokens.get("id_token", ""), nonce, userinfo.get("sub", ""))
    if refusal:
        return await render_template("auth/login.html", error=refusal, oidc_enabled=True), 400
    username_claim = oidc_cfg.get("username_claim", "preferred_username")
    email_claim = oidc_cfg.get("email_claim", "email")
    groups_claim = oidc_cfg.get("groups_claim", "groups")
-    subject = userinfo["sub"]
+    username = userinfo.get(username_claim) or userinfo.get("sub", "")
-    username = userinfo.get(username_claim) or subject
+    email = userinfo.get(email_claim, "") or f"{username}@oidc"
    email = userinfo.get(email_claim, "")
    groups = userinfo.get(groups_claim, [])
    if not isinstance(groups, list):
        groups = []
    role = map_role(groups, oidc_cfg.get("admin_group", ""), oidc_cfg.get("operator_group", ""))
-    user, refusal = await _provision_external_user(current_app, "oidc", subject, username, email, role)
+    if not username:
-    if refusal:
+        return await render_template("auth/login.html",
-        return await _login_refused(refusal, session.pop("oidc_next", None))
+                                     error="SSO returned no username.",
                                     oidc_enabled=True), 400
    user = await _provision_external_user(current_app, username, email, role)
    login_user(user)
    await log_audit(current_app, user.id, user.username, "auth.login",
                    detail={"method": "oidc", "role": role})
@@ -243,15 +178,13 @@ async def login_oidc_callback():
    return redirect(dest)
-@auth_bp.post("/logout")
+@auth_bp.get("/logout")
@public_route("ends the session")
 async def logout():
    logout_user()
    return redirect(url_for("auth.login"))
@auth_bp.get("/setup")
@public_route("first-run setup (refuses once a user exists)")
 async def setup():
    from quart import current_app
    if await get_user_count(current_app) > 0:
@@ -263,7 +196,6 @@ async def setup():
@auth_bp.post("/setup")
@public_route("first-run setup (refuses once a user exists)")
 async def setup_post():
    from quart import current_app
    from steward.core.audit import log_audit
@@ -1,48 +0,0 @@
 """Pause password guessing: after too many failed logins from one address,
 that address's logins are refused for a while.
 Keyed by client address (steward/core/proxy.py: behind a trusted proxy, the
 address it reports), not by username, so nobody can lock a user out by
 failing logins in their name. Held in memory: a restart forgets it, which
 costs an attacker nothing they couldn't get by waiting, and keeps it free of
 the database. The limits are Settings → General (login.max_failures,
 login.lockout_minutes).
 """
 from __future__ import annotations
 import time
 from collections import deque
 class LoginThrottle:
    def __init__(self) -> None:
        self._failures: dict[str, deque[float]] = {}
    def _recent(self, key: str, window_s: float, now: float) -> deque[float]:
        times = self._failures.get(key)
        if times is None:
            return deque()
        while times and times[0] <= now - window_s:
            times.popleft()
        if not times:
            del self._failures[key]
        return times
    def retry_after(self, key: str, max_failures: int, window_s: float,
                    now: float | None = None) -> int:
        """Seconds until `key` may try again; 0 when it may try now."""
        now = time.monotonic() if now is None else now
        times = self._recent(key, window_s, now)
        if len(times) < max_failures:
            return 0
        return max(1, int(times[-max_failures] + window_s - now) + 1)
    def failed(self, key: str, now: float | None = None) -> None:
        now = time.monotonic() if now is None else now
        self._failures.setdefault(key, deque()).append(now)
    def succeeded(self, key: str) -> None:
        self._failures.pop(key, None)
 login_throttle = LoginThrottle()
@@ -49,8 +49,6 @@ def load_bootstrap(config_path: Path | str | None = None) -> dict[str, Any]:
    secret_key = _resolve_secret_key(raw)
    db_connect_timeout = _resolve_db_connect_timeout(raw)
    # Plugin discovery spans two roots (see load_plugins / migration_runner):
    #   • bundled  — first-party plugins shipped inside the image at repo-root
    #     `plugins/`; they version atomically with core and are read-only at runtime.
@@ -70,47 +68,12 @@ def load_bootstrap(config_path: Path | str | None = None) -> dict[str, Any]:
    return {
        "database_url": database_url,
        "secret_key": secret_key,
        "db_connect_timeout": db_connect_timeout,
        "plugin_dirs": plugin_dirs,
        # Installs/downloads target the external (writable, persistent) dir.
        "plugin_install_dir": external_plugin_dir or bundled_plugin_dir,
    }
 def _resolve_db_connect_timeout(raw: dict) -> float:
    """How long to wait for the database at startup, in seconds.
    Bootstrap-only by necessity: this governs reaching the DB, so it cannot
    itself be read from the DB like the rest of Steward's settings.
    A deployment whose database is slower to come up than the default (a large
    cluster replaying WAL, a remote DB behind a link that takes a while) can
    raise it rather than crash-looping the container.
    """
    from .database import DB_CONNECT_TIMEOUT_SECONDS
    value = _env("DB_CONNECT_TIMEOUT") or raw.get("database", {}).get("connect_timeout")
    if value is None or value == "":
        return DB_CONNECT_TIMEOUT_SECONDS
    try:
        parsed = float(value)
    except (TypeError, ValueError):
        logger.warning(
            "Invalid database connect timeout %r — using default %.0fs",
            value, DB_CONNECT_TIMEOUT_SECONDS,
        )
        return DB_CONNECT_TIMEOUT_SECONDS
    if parsed <= 0:
        # 0/negative would mean "never wait", which is the broken behaviour this
        # setting exists to fix — treat it as a mistake, not as an opt-out.
        logger.warning(
            "Database connect timeout %.0fs is not positive — using default %.0fs",
            parsed, DB_CONNECT_TIMEOUT_SECONDS,
        )
        return DB_CONNECT_TIMEOUT_SECONDS
    return parsed
 def _resolve_secret_key(raw: dict) -> str:
    """Resolve secret_key: env var → file → auto-generate.
@@ -2,11 +2,10 @@ from __future__ import annotations
 import asyncio
 import logging
 import uuid
-from dataclasses import dataclass
+from datetime import datetime, timezone
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING
-from sqlalchemy import event as sa_event, select
+from sqlalchemy import and_, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from steward.models.alerts import (
@@ -17,7 +16,6 @@ from steward.models.alerts import (
    AlertStateEnum,
    MaintenanceWindow,
 )
 from steward.core.formatting import human_duration
 from steward.models.metrics import PluginMetric
 if TYPE_CHECKING:
@@ -58,227 +56,82 @@ async def record_metric(
    metric_name: str,
    value: float,
 ) -> None:
-    """One metric sample: record_metrics with a single row."""
+    """Write a metric row and evaluate alert rules inline.
    await record_metrics(session, [PluginMetric(
        source_module=source_module, resource_name=resource_name,
        metric_name=metric_name, value=value)])
-
+    Must be called inside an active transaction:
 class MetricBatch:
    """Gathers one transaction's samples so they are recorded, and their alert
    rules evaluated, in a single record_metrics call:
        batch = MetricBatch()
        batch.add(source_module="unifi", resource_name="wan", metric_name="is_up", value=1.0)
        await batch.record(session)
    """
    def __init__(self) -> None:
        self.rows: list[PluginMetric] = []
    def add(self, source_module: str, resource_name: str, metric_name: str, value: float,
            recorded_at: datetime | None = None) -> None:
        self.rows.append(PluginMetric(source_module=source_module, resource_name=resource_name,
                                      metric_name=metric_name, value=float(value),
                                      recorded_at=recorded_at))
    async def record(self, session: AsyncSession) -> None:
        rows, self.rows = self.rows, []
        await record_metrics(session, rows)
 def _in_window(w: MaintenanceWindow, source_module: str, resource_name: str) -> bool:
    if w.scope_source is None:
        return True
    return w.scope_source == source_module and w.scope_resource in (None, resource_name)
 MUTED = "notification muted: maintenance window"
 RELEASED = "notified when the window ended"
 def _muted_by(windows, rule: AlertRule) -> str | None:
    """The note an event carries when a maintenance window holds back its
    notification, or None when no window covers the rule."""
    w = next((w for w in windows if _in_window(w, rule.source_module, rule.resource_name)), None)
    return f"{MUTED} {w.name}" if w else None
 # Notification and action tasks in flight. Held here so the event loop's weak
 # references are not the only ones (a dropped task is silently never run).
 _background: set[asyncio.Task] = set()
 _AFTER_COMMIT = "steward_alert_after_commit"
 def _spawn(coro) -> None:
    task = asyncio.create_task(coro)
    _background.add(task)
    task.add_done_callback(_background.discard)
 def _after_commit(session: AsyncSession, fn, *args) -> None:
    """Run fn(app, *args) once the session's transaction commits, and never if
    it rolls back: a notification or a playbook run must describe what was
    actually stored."""
    if _app is None:
        return
    info = session.info
    if _AFTER_COMMIT not in info:
        info[_AFTER_COMMIT] = []
        app = _app
        def _committed(_sync_session) -> None:
            jobs, info[_AFTER_COMMIT] = info[_AFTER_COMMIT], []
            for job_fn, job_args in jobs:
                _spawn(job_fn(app, *job_args))
        def _rolled_back(_sync_session) -> None:
            info[_AFTER_COMMIT] = []
        sa_event.listen(session.sync_session, "after_commit", _committed)
        sa_event.listen(session.sync_session, "after_rollback", _rolled_back)
    info[_AFTER_COMMIT].append((fn, args))
 async def record_metrics(session: AsyncSession, rows: list[PluginMetric]) -> None:
    """Write metric rows and evaluate the alert rules that watch them.
    Every ingest path comes through here, so a rule on any metric Steward
    records can fire. Must be called inside an active transaction:
        async with session.begin():
-            await record_metrics(session, rows)
+            await record_metric(session, ...)
-    Maintenance windows, rules and their states are read once per batch, not
+    Notification I/O is deferred via asyncio.create_task() so no network
-    once per row. Each rule sees its key's samples oldest first, so a backlog
+    calls occur while the transaction is open.
    (an agent catching up after an outage) counts consecutive checks as they
    happened. A maintenance window holds back notifications and actions, not
    evaluation: the state still moves, and the event says it was muted.
    Notifications and actions start only once the transaction commits.
    """
    if not rows:
        return
    now = datetime.now(timezone.utc)
-    for row in rows:
+
-        if row.recorded_at is None:
+    # Write metric row
-            row.recorded_at = now
+    metric = PluginMetric(
-    session.add_all(rows)
+        source_module=source_module,
        resource_name=resource_name,
        metric_name=metric_name,
        value=value,
        recorded_at=now,
    )
    session.add(metric)
    await session.flush()
-    by_key: dict[tuple[str, str, str], list[PluginMetric]] = {}
+    # Check active maintenance windows — skip rule evaluation if suppressed
-    for row in rows:
+    mw_result = await session.execute(
-        by_key.setdefault((row.source_module, row.resource_name, row.metric_name), []).append(row)
+        select(MaintenanceWindow).where(
-    sources = {k[0] for k in by_key}
+            MaintenanceWindow.start_at <= now,
-    metric_names = {k[2] for k in by_key}
+            MaintenanceWindow.end_at >= now,
            or_(
                MaintenanceWindow.scope_source.is_(None),
                and_(
                    MaintenanceWindow.scope_source == source_module,
                    or_(
                        MaintenanceWindow.scope_resource.is_(None),
                        MaintenanceWindow.scope_resource == resource_name,
                    ),
                ),
            ),
        ).limit(1)
    )
    if mw_result.scalar_one_or_none() is not None:
        return  # suppressed by maintenance window
-    rules = (await session.execute(
+    # Load matching enabled rules
    result = await session.execute(
        select(AlertRule).where(
-            AlertRule.source_module.in_(sources),
+            AlertRule.source_module == source_module,
-            AlertRule.metric_name.in_(metric_names),
+            AlertRule.resource_name == resource_name,
            AlertRule.metric_name == metric_name,
            AlertRule.enabled.is_(True),
        )
-    )).scalars().all()
+    )
-    rules = [r for r in rules if (r.source_module, r.resource_name, r.metric_name) in by_key]
+    rules = result.scalars().all()
    if not rules:
        return
    windows = (await session.execute(
        select(MaintenanceWindow).where(
            MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
    )).scalars().all()
    states = {st.rule_id: st for st in (await session.execute(
        select(AlertState).where(AlertState.rule_id.in_([r.id for r in rules]))
    )).scalars()}
    for rule in rules:
-        alert_state = states.get(rule.id)
+        state_result = await session.execute(
            select(AlertState).where(AlertState.rule_id == rule.id)
        )
        alert_state = state_result.scalar_one_or_none()
        if alert_state is None:
-            logger.warning("No alert_state row for rule %s, skipping", rule.id)
+            logger.warning(f"No alert_state row for rule {rule.id}, skipping")
            continue
-        samples = sorted(by_key[(rule.source_module, rule.resource_name, rule.metric_name)],
+
-                         key=lambda r: r.recorded_at)
+        notifications = await _evaluate_rule(session, rule, alert_state, value, now)
-        muted = _muted_by(windows, rule)
+
-        resumed = _track_samples(session, rule, alert_state, samples, now, muted)
+        for notif_type, event_id in notifications:
-        if resumed and rule.notify_no_data and not muted:
+            if _app is not None:
-            event_id, value, note = resumed
+                asyncio.create_task(
-            _after_commit(session, _dispatch_notification, "data_resumed", rule, value,
+                    _dispatch_notification(
-                          event_id, note)
+                        _app, notif_type, rule, value, event_id
-        for sample in samples:
+                    )
-            notifications = await _evaluate_rule(session, rule, alert_state, sample.value,
+                )
                                                 sample.recorded_at, note=muted)
            if muted:
                continue
            for notif_type, event_id in notifications:
                _after_commit(session, _dispatch_notification, notif_type, rule, sample.value,
                              event_id)
                # Admin-configured Ansible action runs only on transition-to-firing.
                if notif_type == "firing" and rule.ansible_action:
-                    _after_commit(session, _run_ansible_action, rule, sample.value, event_id)
+                    asyncio.create_task(
-
+                        _run_ansible_action(_app, rule, value, event_id)
-
+                    )
 # A series is silent once it has gone this long without a sample, or three of
 # its usual gaps if that is longer. Until its cadence is known (fewer than two
 # samples seen), it gets an hour: a monitor may check only every ten minutes.
 NO_DATA_FLOOR_S = 300.0
 NO_DATA_GAPS = 3
 NO_DATA_UNKNOWN_S = 3600.0
 def next_interval(interval: float | None, gap: float) -> float:
    """A series' usual gap between samples, updated with one more gap.
    A smoothed average, so one late sample barely moves it. A gap longer than
    the silence threshold counts only as that threshold: an outage is not the
    series' cadence. A real change of cadence still gets learned, a step at a
    time (each step at most 1.5x).
    """
    gap = max(0.0, gap)
    if interval is None or interval <= 0:
        return gap
    return 0.75 * interval + 0.25 * min(gap, NO_DATA_GAPS * interval)
 def no_data_after(interval: float | None) -> float:
    """Seconds without a sample before a series counts as silent."""
    if not interval:
        return NO_DATA_UNKNOWN_S
    return max(NO_DATA_FLOOR_S, NO_DATA_GAPS * interval)
 def silent_since(last_sample_at: datetime | None, interval: float | None,
                 watching_since: datetime, now: datetime) -> datetime | None:
    """When a series went silent, or None while it is still reporting.
    A series that has never had a sample counts from when the rule started
    watching it (created, changed or re-enabled).
    """
    last = last_sample_at or watching_since
    limit = timedelta(seconds=no_data_after(interval))
    return last + limit if now - last > limit else None
 def _track_samples(session: AsyncSession, rule: AlertRule, state: AlertState,
                   samples: list[PluginMetric], now: datetime, muted: str | None = None
                   ) -> tuple[str, float, str] | None:
    """Note a rule's new samples (oldest first) for silence tracking. When the
    series had been marked no_data, clear it and record a resume event,
    returned as (event_id, value, note) for the caller to notify. `muted` is
    added to the event's note when a maintenance window holds it back."""
    for sample in samples:
        if state.last_sample_at is not None:
            if sample.recorded_at <= state.last_sample_at:
                continue  # late or duplicate: not a gap
            gap = (sample.recorded_at - state.last_sample_at).total_seconds()
            state.sample_interval_s = next_interval(state.sample_interval_s, gap)
        state.last_sample_at = sample.recorded_at
    if state.no_data_since is None or not samples:
        return None
    note = f"data resumed after {human_duration((now - state.no_data_since).total_seconds())} silent"
    value = samples[-1].value
    event_id = _add_event(session, rule.id, "no_data", state.state.value, value, now,
                          note=f"{note}; {muted}" if muted else note)
    state.no_data_since = None
    return event_id, value, note
 def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool:
@@ -298,109 +151,80 @@ def _is_breached(value: float, operator: AlertOperator, threshold: float) -> boo
    return False
@dataclass(frozen=True)
 class Step:
    """One sample's effect on an alert: the state after it, the two run
    counters, and the event to record ("firing" / "resolved") if any."""
    state: AlertStateEnum
    fail_count: int
    ok_count: int
    event: str | None = None
 def next_step(state: AlertStateEnum, fail_count: int, ok_count: int, breached: bool,
              fail_required: int, ok_required: int) -> Step:
    """The alert state machine, as a pure function of one sample.
    inactive → pending → firing when `fail_required` breaching checks run in a
    row; a non-breaching check while pending goes back to inactive.
    firing and acknowledged resolve (to inactive, with a resolved event) only
    after `ok_required` non-breaching checks in a row; a breaching check
    resets that run. An acknowledgement holds while the alert is still
    breached: only a resolve clears it.
    """
    fail_required, ok_required = max(1, fail_required), max(1, ok_required)
    S = AlertStateEnum
    if state in (S.firing, S.acknowledged):
        if breached:
            return Step(state, fail_count + 1, 0)
        if ok_count + 1 >= ok_required:
            return Step(S.inactive, 0, 0, "resolved")
        return Step(state, fail_count, ok_count + 1)
    # inactive, pending (and the transient resolved, read as inactive)
    if not breached:
        return Step(S.inactive, 0, 0)
    fails = fail_count + 1 if state == S.pending else 1
    if fails >= fail_required:
        return Step(S.firing, fails, 0, "firing")
    return Step(S.pending, fails, 0)
 async def _evaluate_rule(
    session: AsyncSession,
    rule: AlertRule,
    state: AlertState,
    value: float,
    now: datetime,
    note: str | None = None,
 ) -> list[tuple[str, str]]:
-    """Apply one sample to a rule's state. Returns [(notif_type, event_id)];
+    """Evaluate a single rule against a metric value. Returns list of (notif_type, event_id)."""
-    an event it records carries `note`."""
+    breached = _is_breached(value, rule.operator, rule.threshold)
    from_state = state.state
    step = next_step(from_state, state.consecutive_failure_count, state.consecutive_ok_count,
                     _is_breached(value, rule.operator, rule.threshold),
                     rule.consecutive_failures_required, rule.consecutive_ok_required)
    state.consecutive_failure_count = step.fail_count
    state.consecutive_ok_count = step.ok_count
    notifications: list[tuple[str, str]] = []
-    if step.state != from_state:
+
-        state.state = step.state
+    if state.state == AlertStateEnum.inactive:
-        state.last_transition_at = now
+        if breached:
-        if step.state == AlertStateEnum.inactive:
+            state.consecutive_failure_count = 1
            if state.consecutive_failure_count >= rule.consecutive_failures_required:
                state.state = AlertStateEnum.firing
                state.last_transition_at = now
                event_id = _add_event(session, rule.id, from_state.value, "firing", value, now)
                notifications.append(("firing", event_id))
            else:
                state.state = AlertStateEnum.pending
                state.last_transition_at = now
    elif state.state == AlertStateEnum.pending:
        if breached:
            state.consecutive_failure_count += 1
            if state.consecutive_failure_count >= rule.consecutive_failures_required:
                state.state = AlertStateEnum.firing
                state.last_transition_at = now
                event_id = _add_event(session, rule.id, from_state.value, "firing", value, now)
                notifications.append(("firing", event_id))
        else:
            state.consecutive_failure_count = 0
            state.state = AlertStateEnum.inactive
            state.last_transition_at = now
    elif state.state == AlertStateEnum.firing:
        if not breached:
            # RESOLVED is transient: record event, notify, then immediately go inactive
            event_id = _add_event(session, rule.id, from_state.value, "resolved", value, now)
            notifications.append(("resolved", event_id))
            state.consecutive_failure_count = 0
            state.state = AlertStateEnum.inactive
            state.last_transition_at = now
    elif state.state == AlertStateEnum.acknowledged:
        if not breached:
            event_id = _add_event(session, rule.id, from_state.value, "resolved", value, now)
            notifications.append(("resolved", event_id))
            state.consecutive_failure_count = 0
            state.state = AlertStateEnum.inactive
            state.last_transition_at = now
            state.acknowledged_by = None
            state.acknowledged_at = None
-    if step.event:
+        else:
-        event_id = _add_event(session, rule.id, from_state.value, step.event, value, now,
+            state.consecutive_failure_count += 1
-                              note=note)
+            if state.consecutive_failure_count >= rule.consecutive_failures_required:
-        notifications.append((step.event, event_id))
+                state.state = AlertStateEnum.firing
                state.last_transition_at = now
                event_id = _add_event(session, rule.id, from_state.value, "firing", value, now)
                notifications.append(("firing", event_id))
    await session.flush()
    return notifications
 async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str) -> None:
    """Put a rule's alert back to inactive because the rule itself changed
    (what it watches, its threshold, or it was disabled or re-enabled), and
    start its silence watch over. An alert that was
    firing or acknowledged gets a resolved event carrying `note`; nobody is
    notified, since nothing about the watched resource changed."""
    state = (await session.execute(
        select(AlertState).where(AlertState.rule_id == rule.id))).scalar_one_or_none()
    if state is None:
        return
    now = datetime.now(timezone.utc)
    if state.state in (AlertStateEnum.firing, AlertStateEnum.acknowledged):
        _add_event(session, rule.id, state.state.value, "resolved", None, now, note=note)
    # The watch starts over now: silence is measured from here, against the
    # cadence of whatever the rule watches from now on.
    state.last_transition_at = now
    state.last_sample_at = None
    state.sample_interval_s = None
    state.no_data_since = None
    state.state = AlertStateEnum.inactive
    state.consecutive_failure_count = 0
    state.consecutive_ok_count = 0
    state.acknowledged_by = None
    state.acknowledged_at = None
 def _add_event(
    session: AsyncSession,
    rule_id: str,
    from_state: str,
    to_state: str,
-    value: float | None,
+    value: float,
    now: datetime,
    note: str | None = None,
 ) -> str:
    """Add an AlertEvent row and return its ID."""
    event_id = str(uuid.uuid4())
@@ -411,7 +235,6 @@ def _add_event(
        to_state=to_state,
        metric_value=value,
        transitioned_at=now,
        note=note,
    )
    session.add(event)
    return event_id
@@ -421,9 +244,8 @@ async def _dispatch_notification(
    app: "Quart",
    notif_type: str,
    rule: AlertRule,
-    value: float | None,
+    value: float,
    event_id: str,
    note: str | None = None,
 ) -> None:
    """Run after the transaction commits. Sends notifications and updates the event row."""
    from steward.core.notifications import dispatch_notifications
@@ -436,7 +258,6 @@ async def _dispatch_notification(
        "threshold": rule.threshold,
        "resource": rule.resource_name,
        "source_module": rule.source_module,
        "note": note,
        "timestamp": datetime.now(timezone.utc).isoformat(),
    }
@@ -470,17 +291,32 @@ async def _run_ansible_action(
 ) -> None:
    """Run the rule's configured Ansible playbook (system-triggered).
-    Starts after the evaluation transaction commits. Goes through
+    Runs after the evaluation transaction commits (scheduled via create_task).
-    runner.trigger_run like every other run path, so the playbook and the
+    Creates a system-triggered AnsibleRun, links it to the firing AlertEvent,
-    inventory must be ones the source lists (ansible/guards.py). The firing
+    then executes it with the static params plus injected alert context vars.
    event links to the run, or says why the action did not start.
    """
-    from steward.ansible.runner import trigger_run
+    from steward.ansible import executor, sources as src_module
    from steward.models.ansible import AnsibleRun, AnsibleRunStatus
    action = rule.ansible_action or {}
-    playbook = action.get("playbook") or ""
+    playbook = action.get("playbook")
-    inventory = action.get("inventory") or ""
+    inventory = action.get("inventory")
-    source_name = action.get("source") or ""
+    if not playbook or not inventory:
        logger.error("Ansible action for rule %r missing playbook/inventory", rule.name)
        return
    try:
        srcs = src_module.get_sources(app.config.get("ANSIBLE", {}))
    except Exception:
        logger.exception("Ansible action for rule %r: could not load sources", rule.name)
        return
    source = next((s for s in srcs if s["name"] == action.get("source")), None)
    if source is None:
        logger.error("Ansible action for rule %r: source %r not configured",
                     rule.name, action.get("source"))
        return
    # Static params + auto-injected alert context (all argv, never shell).
    params: dict = {
        "extra_vars": list(action.get("extra_vars") or []) + alert_extra_vars(rule, value),
    }
@@ -490,116 +326,22 @@ async def _run_ansible_action(
        params["tags"] = action["tags"]
    if action.get("check"):
        params["check"] = True
    scope = inventory if inventory.startswith("steward:") else f"repo:{source_name}:{inventory}"
    if not playbook or not inventory:
        run, err = None, "the action has no playbook or inventory"
    else:
        try:
            run, _source, err = await trigger_run(
                app, source_name=source_name, playbook_path=playbook,
                inventory_scope=scope, params=params, triggered_by=None)
        except Exception as exc:
            logger.exception("Ansible action for rule %r failed to start", rule.name)
            run, err = None, str(exc)
    if err:
        logger.error("Ansible action for rule %r not started: %s", rule.name, err)
    run_id = str(uuid.uuid4())
    async with app.db_sessionmaker() as db:
        async with db.begin():
-            ev = (await db.execute(
+            db.add(AnsibleRun(
                id=run_id,
                playbook_path=playbook,
                inventory_path=inventory,
                source_name=source["name"],
                triggered_by=None,
                status=AnsibleRunStatus.running,
                params=params,
            ))
            event = (await db.execute(
                select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none()
-            if ev is None:
+            if event:
-                return
+                event.ansible_run_id = run_id
            if run is not None:
                ev.ansible_run_id = run.id
            else:
                why = f"Ansible action not started: {err}"
                ev.note = f"{ev.note}; {why}" if ev.note else why
-
+    await executor.start_run(app, run_id, playbook, inventory, source["path"], params)
 async def check_silence(app: "Quart") -> None:
    """Mark the enabled rules whose series has gone silent (no_data).
    Run every minute by the core scheduler. A rule is silent once its series
    has had no sample for no_data_after(its usual gap); one that never had a
    sample counts from when the rule started watching. The mark sits beside
    the state, so a firing alert that goes silent stays firing. Each silence
    is recorded once, as an event, and notified once (rule.notify_no_data,
    outside maintenance windows), after the transaction commits.
    """
    now = datetime.now(timezone.utc)
    notify: list[tuple[AlertRule, str, str]] = []
    async with app.db_sessionmaker() as session:
        async with session.begin():
            rows = (await session.execute(
                select(AlertState, AlertRule)
                .join(AlertRule, AlertState.rule_id == AlertRule.id)
                .where(AlertRule.enabled.is_(True), AlertState.no_data_since.is_(None))
            )).all()
            if not rows:
                return
            windows = (await session.execute(
                select(MaintenanceWindow).where(
                    MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
            )).scalars().all()
            for state, rule in rows:
                watching_since = max(rule.created_at, state.last_transition_at)
                since = silent_since(state.last_sample_at, state.sample_interval_s,
                                     watching_since, now)
                if since is None:
                    continue
                state.no_data_since = since
                if state.last_sample_at is None:
                    note = (f"no sample yet, {human_duration((now - watching_since).total_seconds())}"
                            " after the rule started watching")
                else:
                    note = f"no sample for {human_duration((now - state.last_sample_at).total_seconds())}"
                    if state.sample_interval_s:
                        note += f" (usually every {human_duration(state.sample_interval_s)})"
                muted = _muted_by(windows, rule)
                event_id = _add_event(session, rule.id, state.state.value, "no_data", None,
                                      now, note=f"{note}; {muted}" if muted else note)
                if rule.notify_no_data and not muted:
                    notify.append((rule, event_id, note))
    for rule, event_id, note in notify:
        _spawn(_dispatch_notification(app, "no_data", rule, None, event_id, note))
 async def notify_after_maintenance(app: "Quart") -> None:
    """Notify the alerts that started firing inside a maintenance window and
    are still firing now that no window covers them, and run their actions:
    a window delays what it mutes, it does not discard it. Run every minute
    by the core scheduler; each muted firing is released once."""
    now = datetime.now(timezone.utc)
    release: list[tuple[AlertRule, str, float | None]] = []
    async with app.db_sessionmaker() as session:
        async with session.begin():
            rows = (await session.execute(
                select(AlertState, AlertRule)
                .join(AlertRule, AlertState.rule_id == AlertRule.id)
                .where(AlertRule.enabled.is_(True), AlertState.state == AlertStateEnum.firing)
            )).all()
            if not rows:
                return
            windows = (await session.execute(
                select(MaintenanceWindow).where(
                    MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
            )).scalars().all()
            for state, rule in rows:
                if _muted_by(windows, rule):
                    continue
                fired = (await session.execute(
                    select(AlertEvent)
                    .where(AlertEvent.rule_id == rule.id, AlertEvent.to_state == "firing")
                    .order_by(AlertEvent.transitioned_at.desc()).limit(1)
                )).scalar_one_or_none()
                note = (fired.note or "") if fired else ""
                if MUTED not in note or RELEASED in note:
                    continue
                fired.note = f"{note}; {RELEASED}"
                release.append((rule, fired.id, fired.metric_value))
    for rule, event_id, value in release:
        _spawn(_dispatch_notification(app, "firing", rule, value, event_id))
        if rule.ansible_action:
            _spawn(_run_ansible_action(app, rule, value, event_id))
@@ -1,160 +0,0 @@
 """steward/core/api_tokens.py
 Personal API tokens: the credential an AI client (MCP) presents as
 `Authorization: Bearer stw_…` to act as its user.
 Tokens are high-entropy (`secrets.token_urlsafe(32)`), so they are stored as a
 plain sha256 — bcrypt is for low-entropy passwords — the same scheme as
 dashboard share tokens and the host-agent ingest token. The plaintext exists
 only in the create response.
 A token's `role` is a CEILING. Every resolution re-reads the user's current
 role and acts with the lower of the two, so demoting or disabling a user takes
 effect on their tokens at the next request with nothing to revoke by hand.
 """
 from __future__ import annotations
 import hashlib
 import secrets
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from sqlalchemy import select, update
 from steward.auth.middleware import role_meets
 from steward.models.api_tokens import ApiToken
 from steward.models.users import User, UserRole
 TOKEN_PREFIX = "stw_"
 # How many plaintext characters the UI keeps to tell tokens apart.
 _DISPLAY_CHARS = 12
 # last_used_at is a "roughly when" signal; don't write a row per MCP call.
 _LAST_USED_RESOLUTION = timedelta(seconds=60)
@dataclass(frozen=True)
 class Caller:
    """Who an authenticated token request acts as, with its effective role."""
    user_id: str
    username: str
    role: UserRole
    token_id: int
 def generate_token() -> str:
    return TOKEN_PREFIX + secrets.token_urlsafe(32)
 def hash_token(raw: str) -> str:
    return hashlib.sha256(raw.encode()).hexdigest()
 def display_prefix(raw: str) -> str:
    return raw[:_DISPLAY_CHARS]
 def effective_role(ceiling: UserRole, user_role: UserRole) -> UserRole:
    """The lower of a token's ceiling and its user's current role."""
    return ceiling if role_meets(user_role, ceiling) else user_role
 def grantable_roles(user_role: UserRole) -> list[UserRole]:
    """Roles a user may give a new token: their own and anything below it."""
    return [r for r in (UserRole.viewer, UserRole.operator, UserRole.admin)
            if role_meets(user_role, r)]
 def token_state(token: ApiToken, now: datetime | None = None) -> str:
    """'active', 'expired' or 'revoked' — for display and resolution alike."""
    now = now or datetime.now(timezone.utc)
    if token.revoked_at is not None:
        return "revoked"
    if token.expires_at is not None and token.expires_at <= now:
        return "expired"
    return "active"
 async def create_token(
    db, *, user_id: str, user_role: UserRole, name: str, role: UserRole,
    expires_in_days: int | None = None,
 ) -> tuple[str, ApiToken]:
    """Mint a token. Returns (plaintext, row); the plaintext is never stored.
    Raises ValueError when the requested ceiling is above the user's own role
    or the name is empty. The caller owns the transaction and commits it.
    """
    name = (name or "").strip()[:128]
    if not name:
        raise ValueError("Give the token a name, so you can tell it apart later.")
    if not role_meets(user_role, role):
        raise ValueError(
            f"A token can't carry a higher role than yours ({user_role.value})."
        )
    now = datetime.now(timezone.utc)
    raw = generate_token()
    token = ApiToken(
        user_id=user_id,
        name=name,
        token_hash=hash_token(raw),
        prefix=display_prefix(raw),
        role=role,
        created_at=now,
        expires_at=now + timedelta(days=expires_in_days) if expires_in_days else None,
    )
    db.add(token)
    await db.flush()
    return raw, token
 async def list_tokens(db, user_id: str) -> list[ApiToken]:
    result = await db.execute(
        select(ApiToken).where(ApiToken.user_id == user_id)
        .order_by(ApiToken.created_at.desc())
    )
    return list(result.scalars())
 async def revoke_token(db, *, user_id: str, token_id: int) -> ApiToken | None:
    """Revoke one of the user's own tokens. Returns it, or None if not theirs."""
    result = await db.execute(
        select(ApiToken).where(ApiToken.id == token_id, ApiToken.user_id == user_id)
    )
    token = result.scalar_one_or_none()
    if token is not None and token.revoked_at is None:
        token.revoked_at = datetime.now(timezone.utc)
    return token
 async def resolve_token(app, raw: str | None) -> Caller | None:
    """Resolve a bearer token's plaintext to the Caller it acts as.
    None for a missing, unknown, revoked or expired token, or one whose user is
    disabled. Never raises on a bad token; a database failure propagates so
    the transport can report the outage rather than a misleading 401.
    """
    if not raw or not raw.startswith(TOKEN_PREFIX):
        return None
    now = datetime.now(timezone.utc)
    async with app.db_sessionmaker() as db:
        async with db.begin():
            result = await db.execute(
                select(ApiToken, User)
                .join(User, User.id == ApiToken.user_id)
                .where(ApiToken.token_hash == hash_token(raw))
            )
            row = result.first()
            if row is None:
                return None
            token, user = row
            if token_state(token, now) != "active" or not user.is_active:
                return None
            if token.last_used_at is None or now - token.last_used_at >= _LAST_USED_RESOLUTION:
                await db.execute(
                    update(ApiToken).where(ApiToken.id == token.id)
                    .values(last_used_at=now)
                )
            return Caller(
                user_id=user.id,
                username=user.username,
                role=effective_role(token.role, user.role),
                token_id=token.id,
            )
@@ -38,11 +38,3 @@ async def log_audit(
                ))
    except Exception:
        logger.exception("Failed to write audit event action=%r", action)
 async def audit_request(action: str, entity_type: str | None = None,
                        entity_id: str | None = None, detail: dict | None = None) -> None:
    """log_audit for the logged-in user of the current web request."""
    from quart import current_app, session
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    action, entity_type=entity_type, entity_id=entity_id, detail=detail)
@@ -76,14 +76,10 @@ async def _run_docker_retention(session, now: datetime) -> None:
        raw_days = int(await get_setting(session, "docker.retention.metrics_raw_days") or 7)
        rollup_days = int(await get_setting(session, "docker.retention.metrics_rollup_days") or 90)
        events_days = int(await get_setting(session, "docker.retention.events_days") or 30)
        logs_days = int(await get_setting(session, "docker.logs.retention_days") or 3)
        logs_cap = int(
            await get_setting(session, "docker.logs.max_bytes_per_container") or 5_000_000)
        counts = await invoke_capability(
            "docker.run_retention", UserRole.viewer, session,
            events_days=events_days, metrics_raw_days=raw_days,
-            metrics_rollup_days=rollup_days, logs_retention_days=logs_days,
+            metrics_rollup_days=rollup_days, now=now,
            logs_max_bytes_per_container=logs_cap, now=now,
        )
    if counts and any(counts.values()):
        logger.info("Docker retention: %s", counts)
@@ -1,97 +0,0 @@
 """steward/core/formatting.py
 How Steward writes a size, a rate, a span of time and a moment, and how it
 grades a reading against Settings → Thresholds — once, for every page and
 plugin (#5093). Registered as Jinja filters (|bytes, |rate, |duration, |ago)
 and the status_class global in app.py; Python callers import them from here.
 """
 from __future__ import annotations
 from datetime import datetime, timezone
 from typing import Any
 from markupsafe import Markup, escape
 NONE = "—"
 _UNITS = ("B", "KiB", "MiB", "GiB", "TiB", "PiB")
 _SPANS = (("d", 86400), ("h", 3600), ("m", 60), ("s", 1))
 def human_bytes(n: float | None) -> str:
    """Binary units: '0 B', '512 B', '1.5 KiB', '1.4 GiB'."""
    if n is None:
        return NONE
    size = float(n)
    for unit in _UNITS:
        if abs(size) < 1024 or unit == _UNITS[-1]:
            return f"{int(size)} B" if unit == "B" else f"{size:.1f} {unit}"
        size /= 1024
    raise AssertionError("unreachable")
 def human_rate(bytes_per_s: float | None) -> str:
    """Bytes per second in binary units: '12.3 MiB/s'."""
    return NONE if bytes_per_s is None else f"{human_bytes(bytes_per_s)}/s"
 def human_duration(seconds: float | None) -> str:
    """The two largest units of a span: '45s', '8m', '1m 30s', '2h 5m', '3d 4h'.
    The second unit is dropped when it is zero. Negative spans read as None."""
    if seconds is None or seconds < 0:
        return NONE
    left = int(seconds)
    if left == 0:
        return "0s"
    parts = []
    for i, (unit, size) in enumerate(_SPANS):
        count, left = divmod(left, size)
        if count:
            parts.append(f"{count}{unit}")
            nxt = _SPANS[i + 1] if i + 1 < len(_SPANS) else None
            if nxt and left // nxt[1]:
                parts.append(f"{left // nxt[1]}{nxt[0]}")
            break
    return " ".join(parts)
 def _utc(when: datetime) -> datetime:
    return when.replace(tzinfo=timezone.utc) if when.tzinfo is None else when.astimezone(timezone.utc)
 def seconds_since(when: datetime | None, now: datetime | None = None) -> float | None:
    """Seconds from `when` to now; a naive datetime is taken as UTC."""
    if when is None:
        return None
    return ((now or datetime.now(timezone.utc)) - _utc(when)).total_seconds()
 def ago(when: datetime | None, now: datetime | None = None) -> Markup:
    """'3m ago' (or 'in 3m' for a future moment) in a <time> element whose
    title is the absolute UTC time, so a hover gives the exact moment."""
    if when is None:
        return Markup(NONE)
    when = _utc(when)
    secs = seconds_since(when, now)
    if abs(secs) < 1:
        text = "just now"
    elif secs > 0:
        text = f"{human_duration(secs)} ago"
    else:
        text = f"in {human_duration(-secs)}"
    return Markup('<time datetime="{}" title="{}">{}</time>').format(
        when.strftime("%Y-%m-%dT%H:%M:%SZ"), when.strftime("%Y-%m-%d %H:%M:%S UTC"), escape(text))
 def status_class(value: Any, kind: str, thresholds: dict) -> str:
    """Grade a reading against Settings → Thresholds (to_thresholds_cfg):
    'ok', 'warn' or 'crit', or 'unknown' when there is no reading.
    A kind the thresholds don't name is a mistake in the caller, not a
    reading to colour as healthy, so it raises."""
    if value is None:
        return "unknown"
    th = thresholds[kind]
    warn, crit = th["warn"], th["crit"]
    if th.get("dir", "high") == "low":
        return "crit" if value < crit else "warn" if value < warn else "ok"
    return "crit" if value >= crit else "warn" if value >= warn else "ok"
@@ -1,122 +0,0 @@
 """Reads over the generic metric store (plugin_metrics + its hourly rollup).
 Any source that records into PluginMetric — host_agent, traefik, unifi, snmp,
 the monitor types, Steward's self-monitor — is readable through these without
 importing the plugin that wrote it.
 """
 from __future__ import annotations
 from datetime import datetime, timedelta, timezone
 from sqlalchemy import func, or_, select, text
 from steward.core.time_range import bucket_seconds
 from steward.models.metrics import PluginMetric, PluginMetricHourly
 # Raw rows older than this are rolled into plugin_metrics_hourly and deleted
 # (see core/metrics_retention.py), so a range reaching past it reads the rollup.
 RAW_DAYS = 7
 # Loose index scans: step through the distinct leading values of the
 # (source_module, resource_name, …) index one probe at a time instead of
 # scanning every sample. Postgres has no native skip scan before 18.
 _SOURCES_SQL = text("""
    WITH RECURSIVE s(v) AS (
        SELECT min(source_module) FROM plugin_metrics
        UNION ALL
        SELECT (SELECT min(source_module) FROM plugin_metrics WHERE source_module > s.v)
        FROM s WHERE s.v IS NOT NULL
    )
    SELECT v FROM s WHERE v IS NOT NULL
 """)
 _RESOURCES_SQL = text("""
    WITH RECURSIVE r(v) AS (
        SELECT min(resource_name) FROM plugin_metrics WHERE source_module = :src
        UNION ALL
        SELECT (SELECT min(resource_name) FROM plugin_metrics
                WHERE source_module = :src AND resource_name > r.v)
        FROM r WHERE r.v IS NOT NULL
    )
    SELECT v FROM r WHERE v IS NOT NULL
 """)
 async def sources(db) -> list[str]:
    """Every source_module with raw samples on record."""
    return [v for (v,) in await db.execute(_SOURCES_SQL)]
 async def resources(db, source_module: str) -> list[str]:
    """Every resource a source has raw samples for, sorted."""
    return [v for (v,) in await db.execute(_RESOURCES_SQL, {"src": source_module})]
 async def latest_values(db, source_module: str, resource: str, *,
                        include_children: bool = False) -> list[PluginMetric]:
    """The newest sample of each metric for a resource, sorted by (resource, metric).
    include_children also returns the resource's sub-resources, named
    "<resource>:<part>" (a host's mounts, interfaces, sensors).
    """
    match = PluginMetric.resource_name == resource
    if include_children:
        like = resource.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + ":%"
        match = or_(match, PluginMetric.resource_name.like(like, escape="\\"))
    return list((await db.execute(
        select(PluginMetric)
        .where(PluginMetric.source_module == source_module, match)
        .distinct(PluginMetric.resource_name, PluginMetric.metric_name)
        .order_by(PluginMetric.resource_name, PluginMetric.metric_name,
                  PluginMetric.recorded_at.desc())
    )).scalars())
 async def points(db, source_module: str, resource: str, metric: str, since: datetime,
                 *, target: int = 120) -> list[tuple[datetime, float]]:
    """(time, average) points for one metric since `since`, oldest first.
    The part of the range older than the raw window comes from the hourly
    rollup; the rest is bucket-averaged in SQL (buckets no wider than an hour
    when both are present, so the series is evenly grained across the seam).
    """
    now = datetime.now(timezone.utc)
    raw_cutoff = (now - timedelta(days=RAW_DAYS)).replace(minute=0, second=0, microsecond=0)
    out: list[tuple[datetime, float]] = []
    if since < raw_cutoff:
        hourly = await db.execute(
            select(PluginMetricHourly.bucket, PluginMetricHourly.value_avg)
            .where(
                PluginMetricHourly.source_module == source_module,
                PluginMetricHourly.resource_name == resource,
                PluginMetricHourly.metric_name == metric,
                PluginMetricHourly.bucket >= since,
                PluginMetricHourly.bucket < raw_cutoff,
            )
            .order_by(PluginMetricHourly.bucket)
        )
        out.extend((b, float(v)) for b, v in hourly)
    raw_since = max(since, raw_cutoff)
    width_s = bucket_seconds(raw_since, target)
    if since < raw_cutoff:
        width_s = min(width_s, 3600)
    bucket = func.date_bin(
        func.make_interval(0, 0, 0, 0, 0, 0, width_s),
        PluginMetric.recorded_at,
        func.to_timestamp(0),
    ).label("bucket")
    raw = await db.execute(
        select(bucket, func.avg(PluginMetric.value))
        .where(
            PluginMetric.source_module == source_module,
            PluginMetric.resource_name == resource,
            PluginMetric.metric_name == metric,
            PluginMetric.recorded_at >= raw_since,
        )
        .group_by(bucket)
        .order_by(bucket)
    )
    out.extend((b, float(v)) for b, v in raw)
    return out
@@ -8,7 +8,7 @@ import types
 from email.message import EmailMessage
 import httpx
-from jinja2.sandbox import SandboxedEnvironment
+from jinja2 import Template
 logger = logging.getLogger(__name__)
@@ -30,38 +30,20 @@ async def dispatch_notifications(
    return results
 SMTP_TIMEOUT_S = 15
 def smtp_mode(port, tls) -> str:
    """How to talk to the SMTP server: "ssl" (implicit TLS, port 465),
    "starttls" (upgrade a plain connection, when TLS is ticked) or "plain"."""
    if int(port or 0) == 465:
        return "ssl"
    return "starttls" if tls else "plain"
 def email_body(alert: dict) -> str:
    value = "no data" if alert.get("value") is None else alert["value"]
    return (
        f"Alert: {alert['state']}\n"
        f"Rule: {alert['rule_name']}\n"
        f"Resource: {alert['resource']}\n"
        f"Metric: {alert['metric']} = {value}\n"
        f"Threshold: {alert['threshold']}\n"
        + (f"Note: {alert['note']}\n" if alert.get("note") else "")
        + f"Time: {alert['timestamp']}\n"
    )
 async def _send_email(cfg: dict, alert: dict) -> dict:
    try:
        msg = EmailMessage()
-        # A header may not hold a line break; a rule name can (MCP takes any string).
+        msg["Subject"] = f"[Steward] {alert['state']} — {alert['rule_name']}"
-        msg["Subject"] = " ".join(f"[Steward] {alert['state']} — {alert['rule_name']}".split())
+        msg["From"] = cfg.get("username", "steward@localhost")
        msg["From"] = cfg.get("from") or cfg.get("username") or "steward@localhost"
        msg["To"] = ", ".join(cfg["recipients"])
-        msg.set_content(email_body(alert))
+        msg.set_content(
            f"Alert: {alert['state']}\n"
            f"Rule: {alert['rule_name']}\n"
            f"Resource: {alert['resource']}\n"
            f"Metric: {alert['metric']} = {alert['value']}\n"
            f"Threshold: {alert['threshold']}\n"
            f"Time: {alert['timestamp']}\n"
        )
        loop = asyncio.get_running_loop()
        await loop.run_in_executor(None, _smtp_send_sync, cfg, msg)
        return {"sent": True, "error": None}
@@ -71,59 +53,37 @@ async def _send_email(cfg: dict, alert: dict) -> dict:
 def _smtp_send_sync(cfg: dict, msg: EmailMessage) -> None:
-    port = int(cfg.get("port") or 587)
+    ctx = ssl.create_default_context() if cfg.get("tls", True) else None
-    mode = smtp_mode(port, cfg.get("tls", True))
+    with smtplib.SMTP(cfg["host"], cfg.get("port", 587)) as smtp:
-    ctx = ssl.create_default_context()
+        if ctx:
    if mode == "ssl":
        smtp = smtplib.SMTP_SSL(cfg["host"], port, timeout=SMTP_TIMEOUT_S, context=ctx)
    else:
        smtp = smtplib.SMTP(cfg["host"], port, timeout=SMTP_TIMEOUT_S)
    with smtp:
        if mode == "starttls":
            smtp.starttls(context=ctx)
        if cfg.get("username"):
            smtp.login(cfg["username"], cfg.get("password", ""))
        smtp.send_message(msg)
-def _json_str(value) -> str:
+async def _send_webhook(cfg: dict, alert: dict) -> dict:
-    """A value as it may sit between the quotes of a JSON string."""
+    template_str = cfg.get(
-    return json.dumps("" if value is None else str(value))[1:-1]
+        "template",
        '{"content": "{{ alert.state }} — {{ alert.rule_name }}"}',
    )
-
+    # Render Jinja2 template with alert as a namespace object (attribute access)
 def render_webhook(template_str: str, alert: dict) -> str:
    """The webhook body for an alert, checked to be JSON.
    In the template, `alert.<field>` is escaped to sit inside a JSON string
    ("{{ alert.rule_name }}" stays valid whatever the name holds), with a
    missing value reading "no data"; `raw.<field>` is the plain value for use
    with |tojson ({{ raw.value | tojson }} gives a number or null). The
    template runs sandboxed. ValueError says why a body could not be made.
    """
    shown = dict(alert)
    if shown.get("value") is None:
        shown["value"] = "no data"
    env = SandboxedEnvironment()
    try:
-        rendered = env.from_string(template_str).render(
+        alert_ns = types.SimpleNamespace(**alert)
-            alert=types.SimpleNamespace(**{k: _json_str(v) for k, v in shown.items()}),
+        rendered = Template(template_str).render(alert=alert_ns)
            raw=types.SimpleNamespace(**alert))
    except Exception as exc:
-        raise ValueError(f"Template render failed: {exc}") from None
+        error = f"Template render failed: {exc}"
        logger.error(error)
        return {"sent": False, "error": error}
    # Validate JSON before sending
    try:
        json.loads(rendered)
    except json.JSONDecodeError as exc:
-        raise ValueError(f"Rendered webhook template is not valid JSON: {exc}") from None
+        error = f"Rendered webhook template is not valid JSON: {exc}"
-    return rendered
+        logger.error(error)
-
+        return {"sent": False, "error": error}
 async def _send_webhook(cfg: dict, alert: dict) -> dict:
    template_str = cfg.get("template") or '{"content": "{{ alert.state }} — {{ alert.rule_name }}"}'
    try:
        rendered = render_webhook(template_str, alert)
    except ValueError as exc:
        logger.error(str(exc))
        return {"sent": False, "error": str(exc)}
    try:
        async with httpx.AsyncClient(timeout=10) as client:
@@ -135,8 +95,6 @@ async def _send_webhook(cfg: dict, alert: dict) -> dict:
            response.raise_for_status()
        return {"sent": True, "error": None}
    except Exception as exc:
-        # The URL is a secret (it usually carries its own token) and httpx
+        error = str(exc)
        # puts it in its error messages.
        error = str(exc).replace(cfg["url"], "<webhook URL>")
        logger.error(f"Webhook notification failed: {error}")
        return {"sent": False, "error": error}
@@ -3,8 +3,8 @@ from __future__ import annotations
 import hashlib
 import logging
 import os
 import shutil
 import sys
 import tempfile
 import zipfile
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -29,11 +29,6 @@ _FAILED_PLUGINS: dict[str, str] = {}
 #   {"plugin": <name>, "label": str, "href": str, "section": str}
 _PLUGIN_NAV: list[dict] = []
 # Loaded plugin modules, by name, for the MCP server: a plugin contributes
 # agent tools through an optional register_mcp_tools(register) export, which
 # steward/mcp/server.py calls when it mounts (and on hot-reload, below).
 _PLUGIN_MODULES: dict[str, object] = {}
 def get_plugin_failures() -> dict[str, str]:
    """Return a copy of the failed-plugin registry (name → error message)."""
@@ -75,23 +70,6 @@ def _collect_plugin_nav(name: str, module) -> None:
        })
 def get_loaded_plugin_modules() -> dict[str, object]:
    """Loaded plugin modules by name (a copy), for hooks consumed after loading."""
    return dict(_PLUGIN_MODULES)
 def _collect_plugin_module(app: "Quart", name: str, module) -> None:
    """Remember a loaded plugin module; hand it to a live MCP server if mounted.
    At boot the MCP server mounts after plugins load and reads them via
    get_loaded_plugin_modules(); a plugin hot-loaded later is registered here.
    """
    _PLUGIN_MODULES[name] = module
    mcp_server = getattr(app, "mcp_server", None)
    if mcp_server is not None:
        mcp_server.register_plugin(name, module)
 def resolve_plugin_path(plugin_dirs: list[Path], name: str) -> Path | None:
    """Return the first plugin root that contains `name`, else None.
@@ -109,8 +87,7 @@ def _import_plugin(name: str, plugin_path: Path):
    """Load a plugin module by file path, avoiding sys.modules stdlib collisions.
    Using importlib.import_module(name) fails for plugins whose names shadow
-    Python stdlib modules (e.g. a plugin named 'json' or 'http' vs the stdlib
+    Python stdlib modules (e.g. the 'http' plugin vs stdlib's 'http' package).
    package of the same name).
    This helper loads from the filesystem path directly and registers the module
    under a namespaced key so relative imports within the plugin still work.
    """
@@ -286,112 +263,22 @@ def load_plugins(app: "Quart") -> None:
        _FAILED_PLUGINS.pop(name, None)  # clear any stale failure from a previous reload attempt
        _LOADED_PLUGINS.add(name)
        _collect_plugin_nav(name, module)
        _collect_plugin_module(app, name, module)
        logger.info("Plugin %r loaded (v%s)", name, meta.get("version", "?"))
 # Bounds on what an install may bring in: the download itself, and what it
 # unpacks to (a small zip can expand to fill the disk).
 MAX_PLUGIN_DOWNLOAD_BYTES = 50 * 1024 * 1024
 MAX_PLUGIN_UNPACKED_BYTES = 200 * 1024 * 1024
 class PluginInstallError(Exception):
    """An install refused or failed; the message is shown to the admin."""
 def check_plugin_name(name: str) -> None:
    """A plugin is imported by its folder name, so the name must be a plain
    Python identifier — which also keeps it from naming a path."""
    if not (name.isascii() and name.isidentifier()):
        raise PluginInstallError(f"{name!r} is not a valid plugin name")
 def verify_checksum(content: bytes, expected_sha256: str) -> None:
    """Refuse a download whose sha256 is missing from the catalog or differs."""
    if not expected_sha256:
        raise PluginInstallError(
            "The catalog lists no checksum for this plugin, so Steward cannot "
            "tell it is the file the catalog meant. Not installed.")
    actual = hashlib.sha256(content).hexdigest()
    if actual != expected_sha256.strip().lower():
        raise PluginInstallError(f"Checksum mismatch: expected {expected_sha256}, got {actual}")
 def extract_plugin(content: bytes, name: str, dest: Path) -> None:
    """Unpack a plugin zip into `dest`, which must not exist yet.
    One top-level folder whose name starts with the plugin name is stripped:
        name/                         (clean plugin zip)
        name-v1.0.0/                  (release tag archive)
    otherwise the zip is unpacked as it stands. Every member must land inside
    `dest` — an absolute path or a `..` that climbs out refuses the whole zip —
    and the unpacked total is capped.
    """
    import io
    root = dest.resolve()
    with zipfile.ZipFile(io.BytesIO(content)) as zf:
        infos = zf.infolist()
        if sum(i.file_size for i in infos) > MAX_PLUGIN_UNPACKED_BYTES:
            raise PluginInstallError(
                f"Unpacks to more than {MAX_PLUGIN_UNPACKED_BYTES // (1024 * 1024)} MB")
        top_level = {i.filename.split("/")[0] for i in infos}
        prefix = ""
        if len(top_level) == 1:
            candidate = next(iter(top_level))
            if candidate.lower().startswith(name.lower()) and any(
                    i.filename.startswith(candidate + "/") for i in infos):
                prefix = candidate + "/"
        planned: list[tuple[zipfile.ZipInfo, Path]] = []
        for info in infos:
            if prefix and not info.filename.startswith(prefix):
                continue
            rel = info.filename[len(prefix):]
            if not rel:
                continue
            target = (root / rel).resolve()
            if not target.is_relative_to(root) or target == root:
                raise PluginInstallError(f"Refused: {info.filename!r} points outside the plugin folder")
            planned.append((info, target))
        root.mkdir(parents=True)
        for info, target in planned:
            if info.is_dir():
                target.mkdir(parents=True, exist_ok=True)
                continue
            target.parent.mkdir(parents=True, exist_ok=True)
            with zf.open(info) as src, open(target, "wb") as out:
                shutil.copyfileobj(src, out)
 async def _download(url: str) -> bytes:
    import httpx
    chunks: list[bytes] = []
    size = 0
    async with httpx.AsyncClient(timeout=60, follow_redirects=True) as client:
        async with client.stream("GET", url) as resp:
            resp.raise_for_status()
            async for chunk in resp.aiter_bytes():
                size += len(chunk)
                if size > MAX_PLUGIN_DOWNLOAD_BYTES:
                    raise PluginInstallError(
                        f"Download is larger than {MAX_PLUGIN_DOWNLOAD_BYTES // (1024 * 1024)} MB")
                chunks.append(chunk)
    return b"".join(chunks)
 async def download_and_install_plugin(
    app: "Quart",
    name: str,
    download_url: str,
    expected_sha256: str,
 ) -> tuple[bool, str]:
-    """Download a plugin zip named by the catalog, verify its checksum, and
+    """Download a plugin zip, verify its checksum, and extract it to PLUGIN_DIR.
    unpack it into the install dir, replacing any earlier copy whole.
-    The URL and checksum must come from the catalog entry, never from the
+    Returns (success, message). Does NOT hot-reload — call hot_reload_plugin()
-    request. Returns (success, message). Does NOT hot-reload — call
+    after this to activate the plugin without a restart (new plugins only).
    hot_reload_plugin() after this to activate a new plugin without a restart.
    """
    import httpx
    # Downloads always land in the external (writable, persistent) install dir,
    # never the read-only bundled dir.
    plugin_dir = Path(app.config["PLUGIN_INSTALL_DIR"]).resolve()
@@ -403,46 +290,68 @@ async def download_and_install_plugin(
        sys.path.insert(0, plugin_dir_str)
    try:
        check_plugin_name(name)
        if not download_url:
            raise PluginInstallError("The catalog lists no download URL for this plugin")
        logger.info("Downloading plugin %r from %s", name, download_url)
-        try:
+        async with httpx.AsyncClient(timeout=60, follow_redirects=True) as client:
-            content = await _download(download_url)
+            resp = await client.get(download_url)
-        except PluginInstallError:
+            resp.raise_for_status()
-            raise
+        content = resp.content
-        except Exception as exc:
+    except Exception as exc:
-            raise PluginInstallError(f"Download failed: {exc}") from exc
+        msg = f"Download failed: {exc}"
-        verify_checksum(content, expected_sha256)
+        logger.error("Plugin %r: %s", name, msg)
        return False, msg
-        # Unpack beside the live copy, then swap: a failed or refused zip
+    # Verify checksum if provided
-        # leaves the installed plugin untouched, and an update drops files the
+    if expected_sha256:
-        # new version no longer ships.
+        actual = hashlib.sha256(content).hexdigest()
-        plugin_dir.mkdir(parents=True, exist_ok=True)
+        if actual != expected_sha256.lower():
-        dest = plugin_dir / name
+            msg = f"Checksum mismatch: expected {expected_sha256}, got {actual}"
-        staging = plugin_dir / f".{name}.installing"
+            logger.error("Plugin %r: %s", name, msg)
-        old = plugin_dir / f".{name}.replaced"
+            return False, msg
-        for leftover in (staging, old):
+
-            shutil.rmtree(leftover, ignore_errors=True)
+    # Extract the zip
-        try:
+    try:
-            extract_plugin(content, name, staging)
+        with tempfile.TemporaryDirectory() as tmpdir:
-        except PluginInstallError:
+            zip_path = Path(tmpdir) / f"{name}.zip"
-            raise
+            zip_path.write_bytes(content)
-        except Exception as exc:
+            with zipfile.ZipFile(zip_path) as zf:
-            raise PluginInstallError(f"Extraction failed: {exc}") from exc
+                members = zf.namelist()
-        try:
+                # Collect unique top-level names (dirs and loose files)
-            if dest.exists():
+                top_level = {m.split("/")[0] for m in members}
-                dest.rename(old)
+
-            staging.rename(dest)
+                dest = plugin_dir / name
-        except OSError as exc:
+                dest.mkdir(parents=True, exist_ok=True)
-            if old.exists() and not dest.exists():
+
-                old.rename(dest)  # put the previous version back
+                # Detect a single top-level directory to strip. Handles:
-            raise PluginInstallError(f"Could not move the plugin into place: {exc}") from exc
+                #   name/               (clean plugin zip)
-        shutil.rmtree(old, ignore_errors=True)
+                #   name-v1.0.0/        (GitHub release tag archive)
-    except PluginInstallError as exc:
+                #   steward-plugins-main/name/  (whole-repo archive)
-        shutil.rmtree(plugin_dir / f".{name}.installing", ignore_errors=True)
+                # Strategy: if there is exactly one top-level item and it is a
-        logger.error("Plugin %r: %s", name, exc)
+                # directory whose name starts with the plugin name (case-insensitive),
-        return False, str(exc)
+                # strip it. Otherwise extract flat into dest.
                prefix: str | None = None
                if len(top_level) == 1:
                    candidate = list(top_level)[0]
                    if candidate.lower().startswith(name.lower()):
                        prefix = candidate + "/"
                for member in members:
                    if prefix:
                        if not member.startswith(prefix):
                            continue
                        rel = member[len(prefix):]
                    else:
                        rel = member
                    if not rel or rel.endswith("/"):
                        if rel:
                            (dest / rel.rstrip("/")).mkdir(parents=True, exist_ok=True)
                        continue
                    target = dest / rel
                    target.parent.mkdir(parents=True, exist_ok=True)
                    target.write_bytes(zf.read(member))
    except Exception as exc:
        msg = f"Extraction failed: {exc}"
        logger.error("Plugin %r: %s", name, msg)
        return False, msg
    # Run migrations for the newly installed plugin.
    # Include all plugin migration dirs so Alembic can resolve any previously-stamped
@@ -573,7 +482,6 @@ def hot_reload_plugin(app: "Quart", name: str) -> tuple[bool, str]:
    _LOADED_PLUGINS.add(name)
    _collect_plugin_nav(name, module)
    _collect_plugin_module(app, name, module)
    logger.info("Plugin %r hot-reloaded (v%s)", name, meta.get("version", "?"))
    return True, f"Plugin activated (v{meta.get('version', '?')})"
@@ -1,50 +0,0 @@
 """What the request looked like before the reverse proxy.
 Steward serves plain HTTP; TLS ends at the operator's proxy (Traefik, often
 with Cloudflare in front). The proxy reports the original scheme and client
 in X-Forwarded-Proto / X-Forwarded-For, but anyone can send those headers,
 so they are believed only as far as the configured number of trusted proxy
 hops (Settings → General, default 1). 0 means no proxy is trusted.
 Each proxy appends to the header, so the value the nearest trusted proxy
 wrote is the hops-th from the right. HTTPS-only behaviour (the session
 cookie's Secure flag, HSTS) turns on per request from is_https(); nothing
 here ever redirects to https.
 """
 from __future__ import annotations
 from quart import current_app, request
 def trusted_value(header_values: list[str], hops: int) -> str | None:
    """The value written by the outermost trusted proxy, or None when there
    are fewer values than trusted hops (or hops is 0)."""
    if hops <= 0:
        return None
    values = [v.strip() for line in header_values for v in line.split(",") if v.strip()]
    if len(values) < hops:
        return None
    return values[-hops]
 def _hops() -> int:
    try:
        return max(0, int(current_app.config.get("TRUSTED_PROXY_HOPS", 1)))
    except (TypeError, ValueError):
        return 1
 def is_https() -> bool:
    """True when this request reached Steward over HTTPS, directly or as
    reported by a trusted proxy."""
    if request.scheme == "https":
        return True
    proto = trusted_value(request.headers.getlist("X-Forwarded-Proto"), _hops())
    return (proto or "").lower() == "https"
 def client_ip() -> str:
    """The client's address as the outermost trusted proxy saw it, else the
    socket peer."""
    forwarded = trusted_value(request.headers.getlist("X-Forwarded-For"), _hops())
    return forwarded or (request.remote_addr or "")
@@ -1,82 +0,0 @@
 """steward/core/renames.py
 Alert rules, maintenance-window scopes and metric history all key a thing
 by (source, name). When a monitor or host is renamed, carry_rename moves
 every one of them to the new name, in the caller's transaction, so the
 thing's rules keep watching it and its history stays in one piece (#5065).
 """
 from __future__ import annotations
 from sqlalchemy import and_, delete, func, or_, update
 from sqlalchemy.ext.asyncio import AsyncSession
 from steward.models.alerts import AlertRule, MaintenanceWindow
 from steward.models.metrics import PluginMetric, PluginMetricHourly
 # What a host's name keys, and how each source names a part of the host:
 # host_agent "web01:/var", "web01:core3"; docker "web01/nginx".
 HOST_SOURCES: tuple[tuple[str, str | None], ...] = (("host_agent", ":"), ("docker", "/"))
 def _matches(source_col, name_col, sources, name: str):
    """(source, name) itself, or a sub-resource "<name><sep>…" of it."""
    clauses = []
    for source, sep in sources:
        under = [name_col == name]
        if sep:
            # An exact prefix: LIKE would read % or _ in a name as wildcards, and
            # a >=/< range follows the collation, which may ignore punctuation.
            under.append(func.left(name_col, len(name) + len(sep)) == name + sep)
        clauses.append(and_(source_col == source, or_(*under)))
    return or_(*clauses)
 def _renamed(name_col, old: str, new: str):
    return func.concat(new, func.substr(name_col, len(old) + 1))
 async def carry_rename(session: AsyncSession, sources, old: str, new: str) -> dict[str, int]:
    """Point everything keyed by `old` under each (source, separator) in
    `sources` at `new`. A separator also carries the sub-resources named
    "<old><sep>…"; None carries the exact name only.
    The caller has already refused a `new` that a live thing holds, so any
    history under `new` belongs to something deleted: it is dropped first,
    rather than merged into this thing's history. Returns rows moved, by kind.
    """
    if not old or old == new:
        return {}
    sources = tuple(sources)
    for model in (PluginMetric, PluginMetricHourly):
        await session.execute(delete(model).where(
            _matches(model.source_module, model.resource_name, sources, new)))
    moved = {}
    for kind, model, src, res in (
        ("rules", AlertRule, AlertRule.source_module, AlertRule.resource_name),
        ("maintenance_windows", MaintenanceWindow, MaintenanceWindow.scope_source,
         MaintenanceWindow.scope_resource),
        ("metric_rows", PluginMetric, PluginMetric.source_module, PluginMetric.resource_name),
        ("hourly_rows", PluginMetricHourly, PluginMetricHourly.source_module,
         PluginMetricHourly.resource_name),
    ):
        result = await session.execute(
            update(model).where(_matches(src, res, sources, old))
            .values({res: _renamed(res, old, new)})
            .execution_options(synchronize_session=False))
        moved[kind] = result.rowcount
    return moved
 def describe(moved: dict[str, int]) -> str:
    """'2 alert rules, 1 maintenance window and 18,204 metric samples', or ''."""
    parts = []
    for kind, one, many in (("rules", "alert rule", "alert rules"),
                            ("maintenance_windows", "maintenance window", "maintenance windows")):
        if moved.get(kind):
            parts.append(f"{moved[kind]} {one if moved[kind] == 1 else many}")
    samples = moved.get("metric_rows", 0) + moved.get("hourly_rows", 0)
    if samples:
        parts.append(f"{samples:,} metric sample{'' if samples == 1 else 's'}")
    if len(parts) > 1:
        return ", ".join(parts[:-1]) + " and " + parts[-1]
    return "".join(parts)
@@ -11,8 +11,7 @@ from email.message import EmailMessage
 from sqlalchemy import and_, case, func, select
-from steward.alerts import service as alert_svc
+from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertEvent
 from steward.models.alerts import AlertEvent
 from steward.models.metrics import PluginMetric
 from steward.models.monitors import Monitor, MonitorResult
@@ -52,17 +51,24 @@ async def build_report_data(app) -> dict:
        ]
        # ── Active alerts ─────────────────────────────────────────────────────
-        active_rows = await alert_svc.active_alerts(db)
+        active_result = await db.execute(
            select(AlertState, AlertRule)
            .join(AlertRule, AlertState.rule_id == AlertRule.id)
            .where(AlertState.state.in_([
                AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending,
            ]))
            .order_by(AlertState.last_transition_at.desc())
        )
        active_alerts = [
            {
-                "state": alert_svc.alert_label(state).upper(),
+                "state": state.state.value.upper(),
                "name": rule.name,
                "resource": rule.resource_name,
                "metric": rule.metric_name,
                "operator": rule.operator.value,
                "threshold": rule.threshold,
            }
-            for state, rule in active_rows
+            for state, rule in active_result.all()
        ]
        # ── Alert event counts (7d) ───────────────────────────────────────────
@@ -80,28 +80,28 @@ async def record_self_metrics(app) -> None:
        logger.debug("self_monitor: /proc/self/fd unavailable — skipping fd metrics")
        return
-    from .alerts import MetricBatch
+    from .alerts import record_metric
    soft = fd_soft_limit()
    pct = (fds / soft * 100.0) if soft else None
    async with app.db_sessionmaker() as session:
        async with session.begin():
-            batch = MetricBatch()
+            await record_metric(
-            batch.add(
+                session=session,
                source_module="steward",
                resource_name="process",
                metric_name="open_fds",
                value=float(fds),
            )
            if pct is not None:
-                batch.add(
+                await record_metric(
                    session=session,
                    source_module="steward",
                    resource_name="process",
                    metric_name="open_fds_pct",
                    value=pct,
                )
            await batch.record(session)
    if pct is not None and pct >= FD_WARN_PCT:
        if not _warned:
@@ -18,7 +18,6 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
 from collections.abc import Iterable, Mapping
 from datetime import datetime, timezone
 from typing import Any
@@ -40,13 +39,7 @@ DEFAULTS: dict[str, Any] = {
    # General — external URL for install scripts, share links, alert deep-links.
    # Empty = fall back to the current request's Host header.
    "general.public_base_url": "",
    # Reverse proxies in front of Steward whose X-Forwarded-* headers are
    # believed (steward/core/proxy.py). 0 = trust none.
    "general.trusted_proxy_hops": 1,
    "session.lifetime_hours": 8,
    # Failed logins from one address before it is refused for the window.
    "login.max_failures": 5,
    "login.lockout_minutes": 15,
    "data.retention_days": 90,
    "monitors.poll_interval_seconds": 60,
    "smtp.host": "",
@@ -55,7 +48,6 @@ DEFAULTS: dict[str, Any] = {
    "smtp.username": "",
    "smtp.password": "",
    "smtp.recipients": [],
    "smtp.from": "",
    "webhook.url": "",
    "webhook.template": _DEFAULT_WEBHOOK_TEMPLATE,
    "ansible.sources": [],
@@ -86,24 +78,12 @@ DEFAULTS: dict[str, Any] = {
    "thresholds.load_warn": 80, "thresholds.load_crit": 100,
    "thresholds.temp_warn": 70, "thresholds.temp_crit": 85,
    "thresholds.uptime_warn": 99.0, "thresholds.uptime_crit": 95.0,
    # Days left on an HTTPS monitor's certificate (lower is worse), and how
    # many container deaths within an hour count as crash-looping.
    "thresholds.tls_warn_days": 14, "thresholds.tls_crit_days": 3,
    "thresholds.crash_loop_deaths": 3,
    # Docker time-series retention (rule 25 — tunable, no restart). Raw 30s
    # samples are heavy, so keep a short raw window then roll up to hourly
    # averages kept much longer; lifecycle events are light, keep a month.
    "docker.retention.metrics_raw_days": 7,
    "docker.retention.metrics_rollup_days": 90,
    "docker.retention.events_days": 30,
    # Container logs (m79): on by default for every container (operator
    # preference). `exclude` names containers the server drops on ingest; the
    # per-container ring bounds storage (rotate oldest past whichever of ~age or
    # ~bytes hits first — a chatty container just keeps a shorter window).
    "docker.logs.enabled": True,
    "docker.logs.exclude": [],
    "docker.logs.retention_days": 3,
    "docker.logs.max_bytes_per_container": 5_000_000,
    # Host/plugin metrics retention (plugin_metrics): keep a short raw window at
    # the agent's ~30s cadence, then roll up to hourly averages kept much longer.
    "metrics.retention.raw_days": 7,
@@ -118,6 +98,7 @@ DEFAULTS: dict[str, Any] = {
    # Per-plugin yaml config defaults are merged on top at load time.
    "plugin.docker": {"enabled": True},
    "plugin.host_agent": {"enabled": True},
    "plugin.http": {"enabled": True},
    "plugin.snmp": {"enabled": True},
    # OIDC single-sign-on
    "oidc.enabled": False,
@@ -135,7 +116,6 @@ DEFAULTS: dict[str, Any] = {
    "ldap.host": "",
    "ldap.port": 389,
    "ldap.tls": False,
    "ldap.ca_cert": "",
    "ldap.bind_dn": "",
    "ldap.bind_password": "",
    "ldap.base_dn": "",
@@ -161,9 +141,6 @@ SECRET_KEYS: set[str] = {
    "ansible.ssh_private_key",
    "ansible.become_password",
    "ansible.vault_password",
    # A webhook URL carries its own credential (Discord, Slack and ntfy put the
    # token in the path).
    "webhook.url",
 }
@@ -205,31 +182,6 @@ def _is_undecryptable(stored: Any, key: str = "") -> bool:
    )
 # Git sources live as a list under one setting, so their tokens are sealed
 # field by field rather than through SECRET_KEYS.
 SOURCES_KEY = "ansible.sources"
 def seal_source_tokens(sources: Any) -> tuple[Any, int]:
    """Encrypt each source's plaintext http_token; returns (sources, how many)."""
    from steward.core.crypto import encrypt_secret, is_encrypted
    if not isinstance(sources, list):
        return sources, 0
    sealed, count = [], 0
    for src in sources:
        token = src.get("http_token") if isinstance(src, dict) else None
        if isinstance(token, str) and token and not is_encrypted(token):
            src = {**src, "http_token": encrypt_secret(token)}
            count += 1
        sealed.append(src)
    return sealed, count
 def source_token_key(name: str) -> str:
    """How a source's token is named in logs and the undecryptable-secrets banner."""
    return f"{SOURCES_KEY}[{name}].http_token"
 # ─────────────────────────────────────────────────────────────────────────────
 # Async helpers (use inside request handlers / scheduled tasks)
 # ─────────────────────────────────────────────────────────────────────────────
@@ -269,82 +221,6 @@ async def set_setting(session: AsyncSession, key: str, value: Any) -> None:
        _undecryptable_secrets.discard(key)
 async def delete_setting(session: AsyncSession, key: str) -> bool:
    """Remove a stored setting row. Returns True if a row was actually deleted.
    Call in a transaction. Deleting a key that also has a DEFAULT reverts it to
    that default rather than unsetting it — so this only meaningfully *removes*
    a setting when no default declares it.
    """
    result = await session.execute(
        select(AppSetting).where(AppSetting.key == key)
    )
    row = result.scalar_one_or_none()
    if row is None:
        return False
    await session.delete(row)
    return True
 async def get_stored_plugin_names(session: AsyncSession) -> list[str]:
    """Plugin names that have a real stored row, ignoring DEFAULTS.
    Deliberately NOT derived from get_all_settings: that merges DEFAULTS in, and
    a plugin.* key present only as a default is a code-level declaration rather
    than operator data. Offering to "remove" such a key would be a lie — the
    default would simply reassert it on the next load (exactly the bug behind
    the phantom `plugin.http` entry). Only stored rows can actually be cleaned up.
    Returns names only, never values: plugin config can hold credentials, and
    nothing that lists orphans needs to read them.
    """
    prefix = "plugin."
    result = await session.execute(
        select(AppSetting.key).where(AppSetting.key.like(f"{prefix}%"))
    )
    return sorted(
        key[len(prefix):] for key in result.scalars() if key != prefix
    )
 def find_orphaned_plugins(
    stored_names: Iterable[str],
    installed_names: Iterable[str],
    failures: Mapping[str, str] | None = None,
 ) -> list[dict[str, Any]]:
    """Plugins that are configured or failed to load, but are not installed.
    Pure so it can be tested without a DB or a filesystem. "Not installed" is
    intentionally not treated as "safe to delete" — an external plugin under
    /data/plugins can be missing merely because the volume isn't mounted or an
    install failed, so the caller surfaces these for an explicit operator
    decision instead of removing them automatically.
    Driven by the UNION of stored settings and load failures, not by stored
    settings alone. `load_plugins` records a failure for any enabled plugin
    whose directory it cannot find, and the admin banner counts those — so a
    failure with no stored row (a plugin enabled by a DEFAULTS entry) would
    otherwise be counted in the banner and shown nowhere, which is exactly the
    dead end this section exists to close.
    Each row carries `removable`: only a plugin with a real stored row can
    actually be cleaned up. Offering Remove for a default-declared plugin would
    be a lie — deleting nothing, while the default reasserts it on next load.
    """
    installed = set(installed_names)
    failures = dict(failures or {})
    stored = {name for name in stored_names if name}
    candidates = (stored | set(failures)) - installed
    return [
        {
            "name": name,
            "reason": failures.get(name),
            "removable": name in stored,
        }
        for name in sorted(candidates)
    ]
 async def get_all_settings(session: AsyncSession) -> dict[str, Any]:
    """Return flat key→value dict with defaults filled in for missing keys."""
    result = await session.execute(select(AppSetting))
@@ -371,7 +247,6 @@ def to_smtp_cfg(settings: dict[str, Any]) -> dict:
        "username": settings.get("smtp.username", ""),
        "password": settings.get("smtp.password", ""),
        "recipients": settings.get("smtp.recipients", []),
        "from": settings.get("smtp.from", ""),
    }
@@ -403,33 +278,13 @@ def to_ldap_cfg(settings: dict[str, Any]) -> dict:
    return {k[len("ldap."):]: settings.get(k, DEFAULTS[k]) for k in DEFAULTS if k.startswith("ldap.")}
 def _int(settings: dict[str, Any], key: str) -> int:
    try:
        return int(settings.get(key, DEFAULTS[key]))
    except (TypeError, ValueError):
        return int(DEFAULTS[key])
 def to_access_cfg(settings: dict[str, Any]) -> dict:
    """app.config keys for sessions, logins and the trusted proxy."""
    from datetime import timedelta
    hours = max(1, _int(settings, "session.lifetime_hours"))
    return {
        "SESSION_LIFETIME_HOURS": hours,
        "PERMANENT_SESSION_LIFETIME": timedelta(hours=hours),
        "TRUSTED_PROXY_HOPS": max(0, _int(settings, "general.trusted_proxy_hops")),
        "LOGIN_MAX_FAILURES": max(1, _int(settings, "login.max_failures")),
        "LOGIN_LOCKOUT_MINUTES": max(1, _int(settings, "login.lockout_minutes")),
    }
 def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
    """Per-metric (warn, crit, direction) policy for degraded-value coloring.
    Centralizes "what counts as degraded" so templates just name a metric kind
-    via the status_class() and threshold_style() jinja globals. `dir` is
+    via the threshold_style() jinja global. `dir` is "high" (higher is worse,
-    "high" (higher is worse, e.g. CPU) or "low" (lower is worse, e.g. uptime
+    e.g. CPU) or "low" (lower is worse, e.g. uptime %). Latency reuses the
-    %). Latency reuses the existing ping good/warn thresholds.
+    existing ping good/warn thresholds.
    """
    g = settings.get
    return {
@@ -440,23 +295,34 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
        "temp":    {"warn": g("thresholds.temp_warn", 70), "crit": g("thresholds.temp_crit", 85), "dir": "high"},
        "latency": {"warn": g("ping.threshold.good_ms", 50), "crit": g("ping.threshold.warn_ms", 200), "dir": "high"},
        "uptime":  {"warn": g("thresholds.uptime_warn", 99.0), "crit": g("thresholds.uptime_crit", 95.0), "dir": "low"},
        "tls_days": {"warn": g("thresholds.tls_warn_days", 14), "crit": g("thresholds.tls_crit_days", 3), "dir": "low"},
        "crash_loop": {"warn": g("thresholds.crash_loop_deaths", 3), "crit": g("thresholds.crash_loop_deaths", 3), "dir": "high"},
    }
 _STATUS_STYLE = {"warn": "color:var(--yellow);font-weight:500;",
                 "crit": "color:var(--red);font-weight:500;"}
 def threshold_style_for(value: Any, kind: str, thresholds: dict) -> str:
-    """Return an inline-style fragment (amber at warn, red at crit) for a metric,
+    """Return an inline-style fragment (amber at warn, red at crit) for a metric.
-    graded by formatting.status_class against `thresholds` (to_thresholds_cfg).
+
-    Empty string when the reading is normal or missing — drop straight into a
+    The Python twin of the old _macros.metric_style, but reading configurable
    cutoffs from `thresholds` (see to_thresholds_cfg) and handling both
    directions. Empty string when normal/unknown/None — drop straight into a
    span's style="".
    """
-    from steward.core.formatting import status_class
+    if value is None:
-    return _STATUS_STYLE.get(status_class(value, kind, thresholds), "")
+        return ""
    th = thresholds.get(kind)
    if not th:
        return ""
    warn, crit, direction = th["warn"], th["crit"], th.get("dir", "high")
    if direction == "low":
        if value < crit:
            return "color:var(--red);font-weight:700;"
        if value < warn:
            return "color:var(--yellow);font-weight:600;"
    else:
        if value >= crit:
            return "color:var(--red);font-weight:700;"
        if value >= warn:
            return "color:var(--yellow);font-weight:600;"
    return ""
 def to_plugins_cfg(settings: dict[str, Any]) -> dict:
@@ -524,13 +390,6 @@ def migrate_plaintext_secrets(db_url: str) -> int:
                        if isinstance(val, str) and val and not is_encrypted(val):
                            row.value_json = json.dumps(encrypt_secret(val))
                            converted += 1
                    row = (await session.execute(
                        select(AppSetting).where(AppSetting.key == SOURCES_KEY))).scalar_one_or_none()
                    if row is not None:
                        sources, sealed = seal_source_tokens(json.loads(row.value_json))
                        if sealed:
                            row.value_json = json.dumps(sources)
                            converted += sealed
        finally:
            await engine.dispose()
        return converted
@@ -559,14 +418,6 @@ def scan_undecryptable_secrets(db_url: str) -> list[str]:
                for row in result.scalars():
                    if _is_undecryptable(json.loads(row.value_json), row.key):
                        bad.append(row.key)
                row = (await session.execute(
                    select(AppSetting).where(AppSetting.key == SOURCES_KEY))).scalar_one_or_none()
                sources = json.loads(row.value_json) if row is not None else []
                for src in sources if isinstance(sources, list) else []:
                    if isinstance(src, dict):
                        key = source_token_key(src.get("name", "?"))
                        if _is_undecryptable(src.get("http_token"), key):
                            bad.append(key)
        finally:
            await engine.dispose()
        return bad
@@ -6,7 +6,7 @@ from datetime import datetime, timezone, timedelta
 from urllib.parse import urlencode
 from quart import Blueprint, render_template, current_app, abort, redirect, request, session
 from sqlalchemy import select, func, update
-from steward.auth.middleware import public_route, require_role, current_user_id
+from steward.auth.middleware import require_role, current_user_id
 from steward.models.users import UserRole
 from steward.models.hosts import Host
 from steward.models.dashboard import Dashboard, DashboardWidget, DashboardShareToken
@@ -513,7 +513,6 @@ async def revoke_share_token(dash_id: int, token_id: int):
 # ── Public share view (no login required) ────────────────────────────────────
@dashboard_bp.get("/share/<raw_token>")
@public_route("a share link; the token is the credential")
 async def public_share(raw_token: str):
    token_hash = hashlib.sha256(raw_token.encode()).hexdigest()
    now        = datetime.now(timezone.utc)
@@ -1,58 +0,0 @@
 """What a dashboard share link opens.
 A share link's page (/share/<token>) loads each of its dashboard's widgets
 from a fragment URL with `&s=<token>` appended. The token opens exactly those
 URLs, read-only: the same path and the same configured parameters (which host,
 which container). Only the time window a viewer can switch on the widget
 itself (VIEW_PARAMS) may differ. Every other page, and the same widget aimed
 at a different host, still needs a login.
 """
 from __future__ import annotations
 import hashlib
 from datetime import datetime, timezone
 from urllib.parse import parse_qsl
 from sqlalchemy import select
 from steward.models.dashboard import DashboardShareToken
 # Parameters a shared widget may change in place: they pick a time window,
 # never which thing is shown.
 VIEW_PARAMS = frozenset({"hours", "range"})
 def share_allows(widget_urls: list[str], path: str, args) -> bool:
    """True when `path` + `args` (minus the token) is one of `widget_urls`,
    allowing only VIEW_PARAMS to differ."""
    got: dict[str, str] = {}
    for key in args:
        if key == "s":
            continue
        values = args.getlist(key)
        if len(values) != 1:
            return False
        got[key] = values[0]
    for url in widget_urls:
        wpath, _, query = url.partition("?")
        if wpath != path:
            continue
        want = {k: v for k, v in parse_qsl(query, keep_blank_values=True) if k not in VIEW_PARAMS}
        if all(got.get(k) == v for k, v in want.items()) and set(got) <= set(want) | VIEW_PARAMS:
            return True
    return False
 async def share_grants(db, raw_token: str, path: str, args) -> bool:
    """True when `raw_token` is a live share token whose dashboard loads this URL."""
    token_hash = hashlib.sha256(raw_token.encode()).hexdigest()
    share = (await db.execute(
        select(DashboardShareToken).where(DashboardShareToken.token_hash == token_hash)
    )).scalar_one_or_none()
    if share is None:
        return False
    if share.expires_at is not None and share.expires_at <= datetime.now(timezone.utc):
        return False
    from .routes import _get_widgets, _resolve_widgets
    widgets = _resolve_widgets(await _get_widgets(db, share.dashboard_id))
    return share_allows([w["hx_url"] for w in widgets], path, args)
@@ -1,49 +1,10 @@
 from __future__ import annotations
 import asyncio
 import logging
 import time
 from typing import TYPE_CHECKING
-from sqlalchemy import text
+from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import (
    create_async_engine, async_sessionmaker, AsyncEngine, AsyncSession,
 )
 from sqlalchemy.pool import NullPool
 if TYPE_CHECKING:
    from quart import Quart
 logger = logging.getLogger(__name__)
 # Recycle pooled connections after 30 minutes. Nothing in Postgres closes an
 # idle connection by default, but the path to it is not always durable: NAT and
 # conntrack tables drop idle flows, and a Docker network rebuild silently
 # strands existing sockets. Capping connection age means a stranded connection
 # is retired on a timer instead of surfacing as a failed query later.
 POOL_RECYCLE_SECONDS = 1800
 # Total budget for the database to become reachable at startup, and the backoff
 # ceiling between attempts. 60s comfortably covers Postgres WAL recovery after
 # an unclean shutdown (observed ~7s) plus Docker DNS coming up on a host reboot.
 DB_CONNECT_TIMEOUT_SECONDS = 60.0
 _BACKOFF_INITIAL_SECONDS = 0.5
 _BACKOFF_MAX_SECONDS = 5.0
 # Per-REQUEST retry schedule (seconds between attempts), total ~1.75s across 4
 # tries. Far shorter than the startup budget on purpose: nobody is watching a
 # container boot, but somebody is watching this page load. Long enough to ride
 # out a pool blip or a fast reconnect, short enough that a genuinely-down
 # database gets an honest answer instead of a spinner.
 _REQUEST_RETRY_DELAYS = (0.25, 0.5, 1.0)
 # Postgres SQLSTATEs that will never resolve by waiting — retrying these just
 # delays a clear error behind a full timeout budget.
 _FATAL_SQLSTATES = {
    "28P01",  # invalid_password
    "28000",  # invalid_authorization_specification
    "3D000",  # invalid_catalog_name — database does not exist
 }
 def init_db(app: "Quart") -> None:
    """Create async engine and attach db_sessionmaker to app.
@@ -52,162 +13,8 @@ def init_db(app: "Quart") -> None:
    Does not create tables — Alembic handles migrations.
    """
    db_url: str = app.config["DATABASE_URL"]
-    engine = create_async_engine(
+    engine = create_async_engine(db_url, echo=False)
        db_url,
        echo=False,
        # Check a pooled connection is still alive before handing it out. When
        # the database restarts, every connection already in the pool is dead
        # at the socket level; without this, SQLAlchemy only discovers that by
        # failing a real query, so the first operation after a DB restart
        # errors out on whatever triggered it (a request, a scheduled poll).
        # The check is a cheap round-trip and it makes a DB restart invisible.
        pool_pre_ping=True,
        pool_recycle=POOL_RECYCLE_SECONDS,
    )
    app.db_sessionmaker: async_sessionmaker[AsyncSession] = async_sessionmaker(
        engine, expire_on_commit=False
    )
    app._db_engine = engine
 def _fatal_sqlstate(exc: BaseException) -> str | None:
    """Return the SQLSTATE if this error chain carries a non-retryable one."""
    seen: set[int] = set()
    cur: BaseException | None = exc
    while cur is not None and id(cur) not in seen:
        seen.add(id(cur))
        sqlstate = getattr(cur, "sqlstate", None)
        if sqlstate in _FATAL_SQLSTATES:
            return sqlstate
        cur = cur.__cause__ or cur.__context__
    return None
 def _describe(exc: BaseException) -> str:
    """Innermost cause, which is the part that says what actually went wrong.
    SQLAlchemy wraps DBAPI errors several layers deep; the outer message is
    boilerplate, so surface the root for the waiting-for-database log line.
    """
    cur: BaseException = exc
    seen: set[int] = {id(cur)}
    while True:
        nxt = cur.__cause__ or cur.__context__
        if nxt is None or id(nxt) in seen:
            return f"{type(cur).__name__}: {cur}"
        seen.add(id(nxt))
        cur = nxt
 def wait_for_database(
    db_url: str,
    timeout_seconds: float = DB_CONNECT_TIMEOUT_SECONDS,
 ) -> None:
    """Block until the database accepts a connection, or raise after the budget.
    Steward otherwise assumes the database is both resolvable and *ready* the
    first time it asks, which is false in two ordinary situations:
      • Host reboot / full stack restart — the app container can try to resolve
        the `db` service name before Docker's embedded DNS has the record,
        giving `gaierror -2 Name or service not known`.
      • Unclean shutdown — Postgres is listening but still replaying WAL, and
        refuses connections with "the database system is not yet accepting
        connections" until recovery reaches a consistent state.
    compose's `depends_on: service_healthy` covers ordering on a clean `up`,
    but not either of the above. Both are transient and self-healing, so retry
    rather than dumping a traceback and dying. Credential and missing-database
    errors are NOT transient and fail immediately.
    """
    deadline = time.monotonic() + timeout_seconds
    delay = _BACKOFF_INITIAL_SECONDS
    attempt = 0
    last_exc: BaseException | None = None
    while True:
        attempt += 1
        try:
            asyncio.run(_probe(db_url))
        except Exception as exc:  # broad by design — classified just below
            sqlstate = _fatal_sqlstate(exc)
            if sqlstate is not None:
                # Wrong password / missing database: waiting cannot fix it.
                raise
            last_exc = exc
            remaining = deadline - time.monotonic()
            if remaining <= 0:
                break
            logger.warning(
                "Database not ready (attempt %d, %.0fs budget left): %s — "
                "retrying in %.1fs",
                attempt, remaining, _describe(exc), min(delay, remaining),
            )
            time.sleep(min(delay, remaining))
            delay = min(delay * 2, _BACKOFF_MAX_SECONDS)
        else:
            if attempt > 1:
                logger.info("Database ready after %d attempt(s)", attempt)
            return
    detail = f" Last error: {_describe(last_exc)}" if last_exc is not None else ""
    raise RuntimeError(
        f"Database did not become available within {timeout_seconds:.0f}s "
        f"({attempt} attempts).{detail}"
    ) from last_exc
 class DatabaseUnavailable(Exception):
    """The database could not be reached while serving a request."""
 async def ensure_database_reachable(engine: AsyncEngine) -> None:
    """Check out one pooled connection, retrying briefly, or raise.
    Startup readiness (wait_for_database) does not help once the app is already
    serving: if the database goes away underneath a running Steward, the next
    request needs a connection, the pooled ones are dead, and establishing a new
    one fails — which is how a login attempt turned into an opaque 500.
    pool_pre_ping already makes recovery automatic *once the database is back*.
    What it cannot do is wait: while the container is genuinely down, its
    replacement connect fails too. So retry briefly here to ride out a restart,
    then give up and let the caller render an honest "database unavailable"
    page rather than a generic error.
    The budget is deliberately short — a person is waiting on this request, and
    a page that hangs for half a minute is worse than one that says plainly
    what is wrong and offers a retry.
    """
    last_exc: BaseException | None = None
    for attempt, delay in enumerate(_REQUEST_RETRY_DELAYS + (None,)):
        try:
            async with engine.connect():
                if attempt:
                    logger.info(
                        "Database reachable again after %d retry attempt(s)", attempt)
                return
        except (SQLAlchemyError, OSError) as exc:
            # Bad credentials / missing database will never resolve by waiting,
            # and they are not what this guard is for — let them surface.
            if _fatal_sqlstate(exc) is not None:
                raise
            last_exc = exc
            if delay is not None:
                await asyncio.sleep(delay)
    raise DatabaseUnavailable(_describe(last_exc)) from last_exc
 async def _probe(db_url: str) -> None:
    """Open one throwaway connection and round-trip a trivial query.
    Uses its own engine with NullPool: this runs before the app engine exists,
    and a probe connection must never be left in a pool for real work to reuse.
    """
    engine = create_async_engine(db_url, echo=False, poolclass=NullPool)
    try:
        async with engine.connect() as conn:
            await conn.execute(text("SELECT 1"))
    finally:
        await engine.dispose()
@@ -1,16 +1,17 @@
 from __future__ import annotations
 import asyncio
 import uuid
 from datetime import datetime, timedelta, timezone
 from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
-from sqlalchemy import select
+from sqlalchemy import and_, case, select, func
 from steward.ansible import executor, sources as ansible_src
 from steward.auth.middleware import require_role
-from steward.core import renames
+from steward.core.audit import log_audit
 from steward.core.audit import audit_request, log_audit
 from steward.models.ansible import AnsibleRun, AnsibleRunStatus
 from steward.models.hosts import Host
 from steward.models.monitors import Monitor, MonitorResult
 from steward.models.users import UserRole
-from steward.hosts import service as host_svc
+from steward.core.status import _last_n_results, _uptime_by_monitor, _heartbeat_title
 hosts_bp = Blueprint("hosts", __name__, url_prefix="/hosts")
@@ -22,18 +23,235 @@ def _ansible_source_names() -> list[str]:
        return []
 async def _compute_uptime(db) -> dict[str, dict]:
    """Per-host uptime % for 24h/7d/30d, aggregated over the host's monitors.
    Returns {host_id: {"24h": float|None, "7d": float|None, "30d": float|None}}.
    """
    now = datetime.now(timezone.utc)
    cutoff_30d = now - timedelta(days=30)
    cutoff_7d  = now - timedelta(days=7)
    cutoff_24h = now - timedelta(hours=24)
    result = await db.execute(
        select(
            Monitor.host_id,
            func.count(MonitorResult.id).label("total_30d"),
            func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up_30d"),
            func.sum(case((MonitorResult.checked_at >= cutoff_7d, 1), else_=0)).label("total_7d"),
            func.sum(case(
                (and_(MonitorResult.checked_at >= cutoff_7d, MonitorResult.is_up.is_(True)), 1),
                else_=0,
            )).label("up_7d"),
            func.sum(case((MonitorResult.checked_at >= cutoff_24h, 1), else_=0)).label("total_24h"),
            func.sum(case(
                (and_(MonitorResult.checked_at >= cutoff_24h, MonitorResult.is_up.is_(True)), 1),
                else_=0,
            )).label("up_24h"),
        )
        .join(Monitor, Monitor.id == MonitorResult.monitor_id)
        .where(Monitor.host_id.isnot(None), MonitorResult.checked_at >= cutoff_30d)
        .group_by(Monitor.host_id)
    )
    def _pct(up, total):
        return round(float(up) / float(total) * 100, 2) if total else None
    stats: dict[str, dict] = {}
    for row in result:
        stats[row.host_id] = {
            "24h": _pct(row.up_24h, row.total_24h),
            "7d":  _pct(row.up_7d,  row.total_7d),
            "30d": _pct(row.up_30d, row.total_30d),
        }
    return stats
 async def _host_status(db, host_ids: list[str]) -> dict[str, dict]:
    """Per-host status rollup from the latest result of each linked monitor.
    {host_id: {"state": "up"|"down"|"pending", "latency_ms": float|None,
               "count": int}}. State is down if ANY linked monitor's latest
    result is down (worst-case), up if at least one is up, else pending.
    """
    if not host_ids:
        return {}
    rn = func.row_number().over(
        partition_by=MonitorResult.monitor_id, order_by=MonitorResult.checked_at.desc()
    ).label("rn")
    subq = (
        select(MonitorResult.id, rn)
        .join(Monitor, Monitor.id == MonitorResult.monitor_id)
        .where(Monitor.host_id.in_(host_ids), Monitor.enabled.is_(True))
        .subquery()
    )
    rows = (await db.execute(
        select(MonitorResult, Monitor.host_id)
        .join(subq, MonitorResult.id == subq.c.id)
        .join(Monitor, Monitor.id == MonitorResult.monitor_id)
        .where(subq.c.rn == 1)
    )).all()
    out: dict[str, dict] = {}
    for res, host_id in rows:
        d = out.setdefault(host_id, {"state": "pending", "latency_ms": None, "count": 0})
        d["count"] += 1
        if not res.is_up:
            d["state"] = "down"
        elif d["state"] != "down":
            d["state"] = "up"
        if res.is_up and res.response_ms is not None:
            d["latency_ms"] = (res.response_ms if d["latency_ms"] is None
                               else min(d["latency_ms"], res.response_ms))
    return out
 async def _host_monitors(db, host_id: str) -> list[dict]:
    """Display rows for the monitors linked to one host (host hub section)."""
    monitors = (await db.execute(
        select(Monitor).where(Monitor.host_id == host_id).order_by(Monitor.name)
    )).scalars().all()
    ids = [m.id for m in monitors]
    recent = await _last_n_results(db, ids)
    uptime = await _uptime_by_monitor(db, ids)
    data = []
    for m in monitors:
        mrows = recent.get(m.id, [])
        latest = mrows[-1] if mrows else None
        data.append({
            "monitor": m, "latest": latest,
            "heartbeat": [{"state": "up" if r.is_up else "down",
                           "title": _heartbeat_title(m.type, r)} for r in mrows],
            "uptime_pct": uptime.get(m.id, {}).get("24h"),
        })
    return data
 async def _agent_metrics_by_host(db, host_names: list[str]) -> dict[str, dict]:
    """Latest agent cpu/mem per host name from the generic PluginMetric table.
    Read directly (PluginMetric is a core model) so the Hosts list can show an
    at-a-glance agent column without importing the host_agent plugin.
    """
    from steward.models.metrics import PluginMetric
    if not host_names:
        return {}
    wanted = ("cpu_pct", "mem_used_pct")
    subq = (
        select(
            PluginMetric.resource_name, PluginMetric.metric_name,
            func.max(PluginMetric.recorded_at).label("m"),
        )
        .where(
            PluginMetric.source_module == "host_agent",
            PluginMetric.metric_name.in_(wanted),
            PluginMetric.resource_name.in_(host_names),
        )
        .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
    ).subquery()
    rows = (await db.execute(
        select(PluginMetric).join(
            subq,
            (PluginMetric.resource_name == subq.c.resource_name)
            & (PluginMetric.metric_name == subq.c.metric_name)
            & (PluginMetric.recorded_at == subq.c.m),
        )
    )).scalars().all()
    out: dict[str, dict] = {}
    for r in rows:
        out.setdefault(r.resource_name, {})[r.metric_name] = r.value
    return out
 async def _agent_overview_by_host(db, host_names: list[str]) -> dict[str, dict]:
    """Per-host agent glance (cpu/mem/root-disk + freshness) from PluginMetric.
    Core-safe (no host_agent import). Freshness compares the latest host-level
    sample against the plugin's stale window (config), defaulting to 180s.
    """
    from steward.models.metrics import PluginMetric
    if not host_names:
        return {}
    now = datetime.now(timezone.utc)
    stale_after = int(
        (current_app.config.get("PLUGINS", {}).get("host_agent", {}) or {})
        .get("stale_after_seconds", 180))
    resources = list(host_names) + [h + ":/" for h in host_names]
    subq = (
        select(
            PluginMetric.resource_name, PluginMetric.metric_name,
            func.max(PluginMetric.recorded_at).label("m"),
        )
        .where(
            PluginMetric.source_module == "host_agent",
            PluginMetric.resource_name.in_(resources),
            PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "disk_used_pct")),
        )
        .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
    ).subquery()
    rows = (await db.execute(
        select(PluginMetric).join(
            subq,
            (PluginMetric.resource_name == subq.c.resource_name)
            & (PluginMetric.metric_name == subq.c.metric_name)
            & (PluginMetric.recorded_at == subq.c.m),
        ).where(PluginMetric.source_module == "host_agent")
    )).scalars().all()
    out: dict[str, dict] = {}
    for r in rows:
        if r.resource_name.endswith(":/") and r.metric_name == "disk_used_pct":
            out.setdefault(r.resource_name[:-2], {})["disk_root"] = r.value
        elif r.resource_name in host_names:
            d = out.setdefault(r.resource_name, {})
            d[r.metric_name] = r.value
            if d.get("_ts") is None or r.recorded_at > d["_ts"]:
                d["_ts"] = r.recorded_at
    for d in out.values():
        ts = d.pop("_ts", None)
        d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
    return out
 async def _agent_cpu_sparks_by_host(db, host_names: list[str], hours: int = 1) -> dict[str, str]:
    """{host_name: inline-SVG cpu sparkline} over the last `hours` (core-safe).
    A tiny at-a-glance trend in each row — the host-view sparkline, on the widget.
    """
    from steward.models.metrics import PluginMetric
    from steward.core.status import sparkline_svg
    if not host_names:
        return {}
    cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
    rows = (await db.execute(
        select(PluginMetric.resource_name, PluginMetric.value, PluginMetric.recorded_at)
        .where(
            PluginMetric.source_module == "host_agent",
            PluginMetric.metric_name == "cpu_pct",
            PluginMetric.resource_name.in_(host_names),
            PluginMetric.recorded_at >= cutoff,
        )
        .order_by(PluginMetric.recorded_at)
    )).all()
    by_host: dict[str, list[float]] = {}
    for r in rows:
        by_host.setdefault(r.resource_name, []).append(r.value)
    # Amber line to read as "CPU"; only draw when there are a couple of points.
    return {
        name: sparkline_svg(vals[-40:], width=64, height=18, stroke="#c8a840")
        for name, vals in by_host.items() if len(vals) >= 2
    }
@hosts_bp.get("/overview/widget")
@require_role(UserRole.viewer)
 async def overview_widget():
    """Dashboard widget: unified per-host monitor + agent glance, linking to the hub."""
    async with current_app.db_sessionmaker() as db:
        hosts = (await db.execute(select(Host).order_by(Host.name))).scalars().all()
-        host_status = await host_svc.host_status(db, [h.id for h in hosts])
+        host_status = await _host_status(db, [h.id for h in hosts])
-        uptime = await host_svc.compute_uptime(db)
+        uptime = await _compute_uptime(db)
        host_names = [h.name for h in hosts]
-        agent = await host_svc.agent_overview_by_host(
+        agent = await _agent_overview_by_host(db, host_names)
-            db, host_names, host_svc.agent_stale_after_seconds(current_app.config))
+        cpu_sparks = await _agent_cpu_sparks_by_host(db, host_names)
        cpu_sparks = await host_svc.agent_cpu_sparks_by_host(db, host_names)
    return await render_template(
        "hosts/overview_widget.html",
@@ -48,9 +266,9 @@ async def list_hosts():
    async with current_app.db_sessionmaker() as db:
        result = await db.execute(select(Host).order_by(Host.name))
        hosts = result.scalars().all()
-        host_status = await host_svc.host_status(db, [h.id for h in hosts])
+        host_status = await _host_status(db, [h.id for h in hosts])
-        uptime = await host_svc.compute_uptime(db)
+        uptime = await _compute_uptime(db)
-        agent_metrics = await host_svc.agent_metrics_by_host(db, [h.name for h in hosts])
+        agent_metrics = await _agent_metrics_by_host(db, [h.name for h in hosts])
    return await render_template(
        "hosts/list.html",
@@ -73,8 +291,8 @@ async def host_detail(host_id: str):
            select(Host).where(Host.id == host_id))).scalar_one_or_none()
        if host is None:
            return "Not found", 404
-        monitors = await host_svc.host_monitors(db, host_id)
+        monitors = await _host_monitors(db, host_id)
-        uptime = (await host_svc.compute_uptime(db)).get(host_id)
+        uptime = (await _compute_uptime(db)).get(host_id)
        linked_target = (await db.execute(
            select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
            .options(selectinload(AnsibleTarget.groups)))).scalar_one_or_none()
@@ -108,10 +326,6 @@ async def create_host():
    )
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            try:
                await host_svc.check_name_free(db, host.name)
            except ValueError as exc:
                return str(exc), 400
            db.add(host)
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "host.created", entity_type="host", entity_id=host.name,
@@ -144,20 +358,8 @@ async def update_host(host_id: str):
            host = result.scalar_one_or_none()
            if host is None:
                return "Not found", 404
-            old_name, name = host.name, form["name"].strip()
+            host.name = form["name"].strip()
            if name != old_name:
                try:
                    await host_svc.check_name_free(db, name, exclude_id=host.id)
                except ValueError as exc:
                    return str(exc), 400
            host.name = name
            host.address = form["address"].strip()
            # Agent and container metrics, and the rules and windows on them,
            # go by the host's name: move them with it.
            moved = await renames.carry_rename(db, renames.HOST_SOURCES, old_name, name)
    if moved:
        await audit_request("host.renamed", "host", name,
                            {"host_id": host_id, "renamed_from": old_name, "carried": moved})
    return redirect(url_for("hosts.host_detail", host_id=host_id))
@@ -238,11 +440,8 @@ async def run_playbook(host_id: str):
    # Shared parser: discovered var__ fields → extra_vars_map, secret__ → secret_vars
    # (unpersisted), plus tags/check. No --limit — there's exactly one host.
-    from steward.ansible.routes import _parse_run_params, _run_vars, check_run_form
+    from steward.ansible.routes import _parse_run_params
    params_or_none, secret_vars, err = _parse_run_params(form)
    if err:
        return err, 400
    err = check_run_form(form, source, playbook, params_or_none, secret_vars)
    if err:
        return err, 400
@@ -274,9 +473,7 @@ async def run_playbook(host_id: str):
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "ansible.host_run", entity_type="host", entity_id=host.name,
-                    detail={"playbook": playbook, "source": source_name, "run_id": run_id,
+                    detail={"playbook": playbook, "source": source_name})
                            "check": bool((params_or_none or {}).get("check")),
                            "vars": sorted(_run_vars(params_or_none, secret_vars))})
    return redirect(url_for("ansible.run_detail", run_id=run_id))
@@ -303,19 +500,21 @@ async def uptime_page():
    async with current_app.db_sessionmaker() as db:
        result = await db.execute(select(Host).order_by(Host.name))
        hosts = result.scalars().all()
-        uptime = await host_svc.compute_uptime(db)
+        uptime = await _compute_uptime(db)
    return await render_template("hosts/uptime.html", hosts=hosts, uptime=uptime)
@hosts_bp.get("/uptime/widget")
@require_role(UserRole.viewer)
 async def uptime_widget():
    share_token = request.args.get("s")
    async with current_app.db_sessionmaker() as db:
        result = await db.execute(select(Host).order_by(Host.name))
        hosts = result.scalars().all()
-        uptime = await host_svc.compute_uptime(db)
+        uptime = await _compute_uptime(db)
    return await render_template(
        "hosts/uptime_widget.html",
        hosts=hosts,
        uptime=uptime,
        share_token=share_token,
    )
@@ -1,263 +0,0 @@
 """Host queries shared by the Hosts pages, dashboard widgets and MCP tools.
 Every function takes an open session and returns plain data; none reads
 Quart's request context, so the web routes and the /mcp tools (which run
 outside Quart's pipeline) call the same code.
 """
 from __future__ import annotations
 from datetime import datetime, timedelta, timezone
 from sqlalchemy import and_, case, func, select
 from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
 from steward.models.hosts import Host
 from steward.models.monitors import Monitor, MonitorResult
 # Host-level agent metrics read for the at-a-glance views (plus the root
 # mount's disk_used_pct, read from the "<host>:/" sub-resource).
 AGENT_GLANCE_METRICS = ("cpu_pct", "mem_used_pct", "disk_used_pct", "disk_used_pct_worst")
 async def check_name_free(db, name: str, exclude_id: str | None = None) -> None:
    """ValueError when another host already has this name, or it is blank.
    Agent metrics, container metrics and the alert rules on them go by the
    host's name, so two hosts sharing one would share one history.
    """
    if not name:
        raise ValueError("a host needs a name.")
    q = select(Host.id).where(Host.name == name)
    if exclude_id:
        q = q.where(Host.id != exclude_id)
    if (await db.execute(q.limit(1))).first():
        raise ValueError(f"another host is already named {name!r}; pick a different name.")
 def agent_stale_after_seconds(config) -> int:
    """The host_agent plugin's freshness window from app config (default 180s)."""
    return int(((config.get("PLUGINS") or {}).get("host_agent") or {})
               .get("stale_after_seconds", 180))
 async def compute_uptime(db) -> dict[str, dict]:
    """Per-host uptime % for 24h/7d/30d, aggregated over the host's monitors.
    Returns {host_id: {"24h": float|None, "7d": float|None, "30d": float|None}}.
    """
    now = datetime.now(timezone.utc)
    cutoff_30d = now - timedelta(days=30)
    cutoff_7d  = now - timedelta(days=7)
    cutoff_24h = now - timedelta(hours=24)
    result = await db.execute(
        select(
            Monitor.host_id,
            func.count(MonitorResult.id).label("total_30d"),
            func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up_30d"),
            func.sum(case((MonitorResult.checked_at >= cutoff_7d, 1), else_=0)).label("total_7d"),
            func.sum(case(
                (and_(MonitorResult.checked_at >= cutoff_7d, MonitorResult.is_up.is_(True)), 1),
                else_=0,
            )).label("up_7d"),
            func.sum(case((MonitorResult.checked_at >= cutoff_24h, 1), else_=0)).label("total_24h"),
            func.sum(case(
                (and_(MonitorResult.checked_at >= cutoff_24h, MonitorResult.is_up.is_(True)), 1),
                else_=0,
            )).label("up_24h"),
        )
        .join(Monitor, Monitor.id == MonitorResult.monitor_id)
        .where(Monitor.host_id.isnot(None), MonitorResult.checked_at >= cutoff_30d)
        .group_by(Monitor.host_id)
    )
    def _pct(up, total):
        return round(float(up) / float(total) * 100, 2) if total else None
    stats: dict[str, dict] = {}
    for row in result:
        stats[row.host_id] = {
            "24h": _pct(row.up_24h, row.total_24h),
            "7d":  _pct(row.up_7d,  row.total_7d),
            "30d": _pct(row.up_30d, row.total_30d),
        }
    return stats
 async def host_status(db, host_ids: list[str]) -> dict[str, dict]:
    """Per-host status rollup from the latest result of each linked monitor.
    {host_id: {"state": "up"|"down"|"pending", "latency_ms": float|None,
               "count": int}}. State is down if ANY linked monitor's latest
    result is down (worst-case), up if at least one is up, else pending.
    """
    if not host_ids:
        return {}
    rn = func.row_number().over(
        partition_by=MonitorResult.monitor_id, order_by=MonitorResult.checked_at.desc()
    ).label("rn")
    subq = (
        select(MonitorResult.id, rn)
        .join(Monitor, Monitor.id == MonitorResult.monitor_id)
        .where(Monitor.host_id.in_(host_ids), Monitor.enabled.is_(True))
        .subquery()
    )
    rows = (await db.execute(
        select(MonitorResult, Monitor.host_id)
        .join(subq, MonitorResult.id == subq.c.id)
        .join(Monitor, Monitor.id == MonitorResult.monitor_id)
        .where(subq.c.rn == 1)
    )).all()
    out: dict[str, dict] = {}
    for res, host_id in rows:
        d = out.setdefault(host_id, {"state": "pending", "latency_ms": None, "count": 0})
        d["count"] += 1
        if not res.is_up:
            d["state"] = "down"
        elif d["state"] != "down":
            d["state"] = "up"
        if res.is_up and res.response_ms is not None:
            d["latency_ms"] = (res.response_ms if d["latency_ms"] is None
                               else min(d["latency_ms"], res.response_ms))
    return out
 async def host_monitors(db, host_id: str) -> list[dict]:
    """Display rows for the monitors linked to one host (host hub section)."""
    monitors = (await db.execute(
        select(Monitor).where(Monitor.host_id == host_id).order_by(Monitor.name)
    )).scalars().all()
    ids = [m.id for m in monitors]
    recent = await _last_n_results(db, ids)
    uptime = await _uptime_by_monitor(db, ids)
    data = []
    for m in monitors:
        mrows = recent.get(m.id, [])
        latest = mrows[-1] if mrows else None
        data.append({
            "monitor": m, "latest": latest,
            "heartbeat": [{"state": "up" if r.is_up else "down",
                           "title": _heartbeat_title(m.type, r)} for r in mrows],
            "uptime_pct": uptime.get(m.id, {}).get("24h"),
        })
    return data
 async def agent_metrics_by_host(db, host_names: list[str]) -> dict[str, dict]:
    """Latest agent cpu/mem per host name from the generic PluginMetric table.
    Read directly (PluginMetric is a core model) so the Hosts list can show an
    at-a-glance agent column without importing the host_agent plugin.
    """
    from steward.models.metrics import PluginMetric
    if not host_names:
        return {}
    wanted = ("cpu_pct", "mem_used_pct")
    subq = (
        select(
            PluginMetric.resource_name, PluginMetric.metric_name,
            func.max(PluginMetric.recorded_at).label("m"),
        )
        .where(
            PluginMetric.source_module == "host_agent",
            PluginMetric.metric_name.in_(wanted),
            PluginMetric.resource_name.in_(host_names),
        )
        .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
    ).subquery()
    rows = (await db.execute(
        select(PluginMetric).join(
            subq,
            (PluginMetric.resource_name == subq.c.resource_name)
            & (PluginMetric.metric_name == subq.c.metric_name)
            & (PluginMetric.recorded_at == subq.c.m),
        )
    )).scalars().all()
    out: dict[str, dict] = {}
    for r in rows:
        out.setdefault(r.resource_name, {})[r.metric_name] = r.value
    return out
 async def agent_overview_by_host(
    db, host_names: list[str], stale_after_seconds: int = 180,
 ) -> dict[str, dict]:
    """Per-host agent glance (cpu/mem/disk + freshness) from PluginMetric.
    {host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root,
                 last_at, fresh}} for each host with any agent samples.
    Core-safe (no host_agent import). Freshness compares the latest host-level
    sample against the host_agent plugin's stale window (stale_after_seconds;
    callers pass agent_stale_after_seconds(app.config)).
    """
    from steward.models.metrics import PluginMetric
    if not host_names:
        return {}
    now = datetime.now(timezone.utc)
    stale_after = int(stale_after_seconds)
    resources = list(host_names) + [h + ":/" for h in host_names]
    subq = (
        select(
            PluginMetric.resource_name, PluginMetric.metric_name,
            func.max(PluginMetric.recorded_at).label("m"),
        )
        .where(
            PluginMetric.source_module == "host_agent",
            PluginMetric.resource_name.in_(resources),
            PluginMetric.metric_name.in_(AGENT_GLANCE_METRICS),
        )
        .group_by(PluginMetric.resource_name, PluginMetric.metric_name)
    ).subquery()
    rows = (await db.execute(
        select(PluginMetric).join(
            subq,
            (PluginMetric.resource_name == subq.c.resource_name)
            & (PluginMetric.metric_name == subq.c.metric_name)
            & (PluginMetric.recorded_at == subq.c.m),
        ).where(PluginMetric.source_module == "host_agent")
    )).scalars().all()
    out: dict[str, dict] = {}
    for r in rows:
        if r.resource_name.endswith(":/") and r.metric_name == "disk_used_pct":
            out.setdefault(r.resource_name[:-2], {})["disk_root"] = r.value
        elif r.resource_name in host_names:
            d = out.setdefault(r.resource_name, {})
            d[r.metric_name] = r.value
            if d.get("_ts") is None or r.recorded_at > d["_ts"]:
                d["_ts"] = r.recorded_at
    for d in out.values():
        ts = d.pop("_ts", None)
        d["last_at"] = ts
        d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
    return out
 async def agent_cpu_sparks_by_host(db, host_names: list[str], hours: int = 1) -> dict[str, str]:
    """{host_name: inline-SVG cpu sparkline} over the last `hours` (core-safe).
    A tiny at-a-glance trend in each row — the host-view sparkline, on the widget.
    """
    from steward.models.metrics import PluginMetric
    from steward.core.status import sparkline_svg
    if not host_names:
        return {}
    cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
    rows = (await db.execute(
        select(PluginMetric.resource_name, PluginMetric.value, PluginMetric.recorded_at)
        .where(
            PluginMetric.source_module == "host_agent",
            PluginMetric.metric_name == "cpu_pct",
            PluginMetric.resource_name.in_(host_names),
            PluginMetric.recorded_at >= cutoff,
        )
        .order_by(PluginMetric.recorded_at)
    )).all()
    by_host: dict[str, list[float]] = {}
    for r in rows:
        by_host.setdefault(r.resource_name, []).append(r.value)
    # Amber line to read as "CPU"; only draw when there are a couple of points.
    return {
        name: sparkline_svg(vals[-40:], width=64, height=18, stroke="#c8a840")
        for name, vals in by_host.items() if len(vals) >= 2
    }
@@ -1,4 +0,0 @@
 """The MCP endpoint (/mcp): Steward as a server an LLM client can inspect and drive."""
 from .server import mount_mcp
 __all__ = ["mount_mcp"]
@@ -1,50 +0,0 @@
 """Per-request MCP context: who is calling, and which app they are calling.
 The /mcp transport bypasses Quart's request pipeline, so neither `session`
 nor `current_app` exists inside a tool. The ASGI auth wrapper in server.py
 sets these from the resolved bearer token before dispatching; tools read them
 through `current_caller()` and `mcp_app()`.
 Neither falls back to a default: a tool reached without an authenticated
 caller is a bug in the transport, and it raises rather than acting for nobody.
 """
 from __future__ import annotations
 from contextvars import ContextVar
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
    from quart import Quart
    from steward.core.api_tokens import Caller
 _caller_ctx: ContextVar["Caller | None"] = ContextVar("steward_mcp_caller", default=None)
 _app_ctx: ContextVar["Quart | None"] = ContextVar("steward_mcp_app", default=None)
 # The scheme and host the caller reached /mcp on, e.g. "https://steward.example".
 _base_url_ctx: ContextVar[str] = ContextVar("steward_mcp_base_url", default="")
 def current_caller() -> "Caller":
    caller = _caller_ctx.get()
    if caller is None:
        raise RuntimeError("no MCP caller in context: the request was not bearer-authenticated")
    return caller
 def current_caller_or_none() -> "Caller | None":
    return _caller_ctx.get()
 def mcp_app() -> "Quart":
    app = _app_ctx.get()
    if app is None:
        raise RuntimeError("no Steward app in MCP context")
    return app
 def public_url() -> str:
    """Steward's externally reachable base URL, as the web side's
    public_base_url() gives it: Settings → General → Public base URL when set,
    else the scheme and host this MCP request arrived on; "" when neither is
    known."""
    app = _app_ctx.get()
    configured = (app.config.get("PUBLIC_BASE_URL") or "").strip() if app is not None else ""
    return (configured or _base_url_ctx.get()).rstrip("/")
@@ -1,302 +0,0 @@
 """Steward's MCP server: the StewardMCPServer instance and its mount at /mcp.
 The /mcp path is dispatched to the MCP SDK's streamable-HTTP ASGI app BEFORE
 Quart sees it (wrapping `app.asgi_app`), so Quart's session, before_request
 hooks and error pages do not apply. This module supplies the equivalents:
 - bearer auth: a personal API token resolves to a Caller (user + effective
  role), set in a ContextVar for the tools — 401 otherwise;
 - the database gate: the same ride-out-a-restart probe the web pipeline uses,
  answering 503 rather than failing deep in a tool;
 - role gating, default-deny: every tool is registered with a minimum role and
  a tool the caller's role doesn't reach is neither listed nor callable.
 """
 from __future__ import annotations
 import difflib
 import functools
 import json
 import logging
 from mcp.server.mcpserver import MCPServer
 from mcp.server.mcpserver.exceptions import ToolError, UnexpectedToolError
 from mcp.server.transport_security import TransportSecuritySettings
 from mcp.types import ToolAnnotations
 from sqlalchemy.exc import SQLAlchemyError
 from steward.auth.middleware import role_meets
 from steward.models.users import UserRole
 from . import _context
 logger = logging.getLogger(__name__)
 MCP_PATH = "/mcp"
 # Claude Code injects only the first ~2,048 characters of a server's
 # instructions; keep this well under (test_instructions_fit). The workflow
 # across tools lives here; each tool's own contract lives in its docstring.
 INSTRUCTIONS = """\
 Steward is a self-hosted monitoring and infrastructure hub: hosts, uptime
 monitors, host-agent metrics, Docker containers and logs, alerts, and Ansible
 automation. You act as the user who created your token, at that token's role.
 Start broad, then narrow: get_overview says what is wrong right now; follow
 its ids into get_host / get_monitor / get_container, then metrics and logs.
 List tools are bounded: a response with `truncated: true` says which argument
 fetches more. Times are UTC ISO-8601; units are in field names (_pct, _ms,
 _bytes).
 Changes are audited under the user's name. run_playbook is a dry run (Ansible
 check mode) unless you pass check=false; dry-run first and show the user the
 diff before applying. If a tool refuses for lack of role, whoami says what
 this token can do.
 """
 _DB_UNAVAILABLE = (
    "Steward is running but can't reach its database right now, so nothing was "
    "read or changed. This is usually a database restart; retry in a minute."
 )
 def _is_db_failure(exc: BaseException | None) -> bool:
    seen: set[int] = set()
    while exc is not None and id(exc) not in seen:
        seen.add(id(exc))
        if isinstance(exc, (SQLAlchemyError, OSError, TimeoutError)):
            return True
        exc = exc.__cause__ or exc.__context__
    return False
 class StewardMCPServer(MCPServer):
    """MCPServer with per-tool minimum roles, plugin-scoped tools and strict args."""
    def __init__(self, app, **kwargs):
        super().__init__("steward", instructions=INSTRUCTIONS.strip(), **kwargs)
        self._steward_app = app
        self.tool_roles: dict[str, UserRole] = {}
        self.tool_plugins: dict[str, str] = {}
        # plugin name -> its `mcp_overview(db)`: findings it adds to get_overview.
        self.overview_sources: dict[str, object] = {}
    # ── registration ─────────────────────────────────────────────────────────
    def register_tool(self, fn, *, role: UserRole, read_only: bool, plugin: str | None = None) -> None:
        """The only way a tool reaches the server: name, minimum role, effect.
        `role` and `read_only` are required — there is no default for a
        mutating tool to inherit by omission.
        """
        name = fn.__name__
        if name in self.tool_roles:
            raise ValueError(f"MCP tool {name!r} is already registered")
        if not isinstance(role, UserRole):
            raise TypeError(f"MCP tool {name!r}: role must be a UserRole, got {role!r}")
        self.add_tool(
            fn, name=name,
            annotations=ToolAnnotations(read_only_hint=read_only,
                                        destructive_hint=None if read_only else True),
        )
        self.tool_roles[name] = role
        if plugin:
            self.tool_plugins[name] = plugin
    def register_plugin(self, name: str, module) -> None:
        """Add a plugin's tools via its optional `register_mcp_tools(register)`,
        and its optional `mcp_overview(db)` as a get_overview findings source.
        Idempotent per plugin (hot-reload drops its previous tools first) and
        tolerant: a raising hook is logged and leaves the plugin with no tools,
        never a broken server.
        """
        self._drop_plugin_tools(name)
        self.overview_sources.pop(name, None)
        overview = getattr(module, "mcp_overview", None)
        if callable(overview):
            self.overview_sources[name] = overview
        hook = getattr(module, "register_mcp_tools", None)
        if hook is None:
            return
        try:
            hook(functools.partial(self.register_tool, plugin=name))
        except Exception:
            # All or nothing: a hook that fails halfway leaves no partial set.
            self._drop_plugin_tools(name)
            logger.exception("Plugin %r: register_mcp_tools() raised; its MCP tools are unavailable", name)
    def _drop_plugin_tools(self, name: str) -> None:
        for tool_name in [t for t, p in self.tool_plugins.items() if p == name]:
            self.remove_tool(tool_name)
            self.tool_roles.pop(tool_name, None)
            self.tool_plugins.pop(tool_name, None)
    # ── gating ───────────────────────────────────────────────────────────────
    def plugin_enabled(self, plugin: str) -> bool:
        cfg = (self._steward_app.config.get("PLUGINS") or {}).get(plugin)
        return isinstance(cfg, dict) and bool(cfg.get("enabled"))
    def refusal(self, name: str, caller) -> str | None:
        """Why `caller` may not use tool `name`, or None if they may."""
        required = self.tool_roles.get(name)
        if required is None:
            return f"There is no Steward tool named {name!r}."
        plugin = self.tool_plugins.get(name)
        if plugin and not self.plugin_enabled(plugin):
            return (f"{name} belongs to the {plugin} plugin, which is disabled in this "
                    "Steward. An admin can enable it under Settings → Plugins.")
        if caller is None:
            return "This request is not authenticated."
        if not role_meets(caller.role, required):
            return (f"{name} needs the {required.value} role; this connection acts as "
                    f"{caller.role.value}. A token's role is capped at its user's role — "
                    "create a token with a higher role on Steward's AI access page if "
                    "your account allows it.")
        return None
    async def list_tools(self):
        caller = _context.current_caller_or_none()
        tools = await super().list_tools()
        return [t for t in tools if self.refusal(t.name, caller) is None]
    async def call_tool(self, name, arguments, context=None):
        refusal = self.refusal(name, _context.current_caller_or_none())
        if refusal:
            raise ToolError(refusal)
        # Reject undeclared arguments: pydantic would silently drop a misnamed
        # one and run the tool with that field's default.
        tool = self._tool_manager.get_tool(name)
        declared = set((tool.parameters or {}).get("properties", {})) if tool else set()
        unknown = sorted(set(arguments or {}) - declared)
        if unknown:
            hints = []
            for arg in unknown:
                close = difflib.get_close_matches(arg, sorted(declared), n=1)
                hints.append(f"'{arg}'" + (f" (did you mean '{close[0]}'?)" if close else ""))
            raise ToolError(
                f"{name} does not accept {', '.join(hints)}. It accepts: "
                f"{', '.join(sorted(declared)) or 'no arguments'}. Nothing was run."
            )
        try:
            return await super().call_tool(name, arguments, context)
        except UnexpectedToolError as exc:
            cause = exc.__cause__
            # ValueError is how a Steward tool refuses: what, why, what instead.
            if isinstance(cause, ValueError):
                raise ToolError(f"{name}: {cause}") from cause
            if _is_db_failure(cause):
                logger.warning("MCP tool %s hit a database failure: %r", name, cause)
                raise ToolError(_DB_UNAVAILABLE) from cause
            raise
 def build_mcp_server(app) -> StewardMCPServer:
    from .tools import register_all
    mcp = StewardMCPServer(app)
    register_all(mcp.register_tool)
    return mcp
 async def _send_json(send, status: int, payload: dict, extra_headers=()) -> None:
    await send({
        "type": "http.response.start",
        "status": status,
        "headers": [(b"content-type", b"application/json"), *extra_headers],
    })
    await send({"type": "http.response.body", "body": json.dumps(payload).encode()})
 def mount_mcp(app) -> StewardMCPServer:
    """Mount the MCP endpoint at /mcp and return the server.
    stateless_http: every request stands alone on its bearer token. The
    stateful default strands Claude Code after a redeploy — it reconnects with
    a session id the new process never issued and won't re-initialize on the
    404 — so stateless is what makes a restart invisible to clients.
    DNS-rebinding protection is off: its default allow-list is localhost only,
    so any deployment behind a reverse proxy would get 421s. That protection
    guards a browser rebinding DNS onto a localhost server; here the bearer
    token is the security boundary.
    """
    # Looked up through their modules per request (not bound here) so a test
    # can stand in for the token store or the database probe.
    from steward import database
    from steward.core import api_tokens
    from steward.core.plugin_manager import get_loaded_plugin_modules
    mcp = build_mcp_server(app)
    for name, module in sorted(get_loaded_plugin_modules().items()):
        mcp.register_plugin(name, module)
    mcp_asgi = mcp.streamable_http_app(
        streamable_http_path=MCP_PATH,
        stateless_http=True,
        transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False),
    )
    app.mcp_server = mcp
    # The SDK's session manager owns a task group that must be running before
    # it serves; host it in Quart's serving lifecycle.
    @app.before_serving
    async def _start_mcp() -> None:
        cm = mcp.session_manager.run()
        await cm.__aenter__()
        app._mcp_session_cm = cm
    @app.after_serving
    async def _stop_mcp() -> None:
        cm = getattr(app, "_mcp_session_cm", None)
        if cm is not None:
            await cm.__aexit__(None, None, None)
    async def handle_mcp(scope, receive, send):
        headers = {k.decode("latin-1").lower(): v.decode("latin-1")
                   for k, v in scope.get("headers", [])}
        auth = headers.get("authorization", "")
        raw = auth[7:].strip() if auth[:7].lower() == "bearer " else ""
        engine = getattr(app, "_db_engine", None)
        try:
            if engine is not None:
                await database.ensure_database_reachable(engine)
            caller = await api_tokens.resolve_token(app, raw)
        except (database.DatabaseUnavailable, SQLAlchemyError, OSError) as exc:
            logger.error("MCP request refused, database unreachable: %r", exc)
            await _send_json(send, 503, {"error": "database_unavailable", "message": _DB_UNAVAILABLE})
            return
        if caller is None:
            await _send_json(
                send, 401,
                {"error": "unauthorized",
                 "message": "Send a Steward API token as 'Authorization: Bearer stw_…'. "
                            "Create one on Steward's AI access page."},
                extra_headers=[(b"www-authenticate", b'Bearer realm="steward-mcp"')],
            )
            return
        caller_token = _context._caller_ctx.set(caller)
        app_token = _context._app_ctx.set(app)
        host = headers.get("host", "")
        url_token = _context._base_url_ctx.set(f"{scope.get('scheme', 'http')}://{host}" if host else "")
        try:
            await mcp_asgi(scope, receive, send)
        finally:
            _context._caller_ctx.reset(caller_token)
            _context._app_ctx.reset(app_token)
            _context._base_url_ctx.reset(url_token)
    original_asgi = app.asgi_app
    async def dispatch(scope, receive, send):
        if scope["type"] == "http":
            path = scope.get("path", "")
            if path == MCP_PATH or path.startswith(MCP_PATH + "/"):
                # Path passes through untouched: the SDK app routes on /mcp itself.
                return await handle_mcp(scope, receive, send)
        return await original_asgi(scope, receive, send)
    app.asgi_app = dispatch
    return mcp
@@ -1,22 +0,0 @@
 """Core MCP tool modules.
 Each module defines plain `async def` tools at module level (tests call them
 directly) and a `register(register_tool)` that lists them with their minimum
 role. `register_tool(fn, role=..., read_only=...)` takes both as required
 keywords: a tool with no declared role is unreachable, and there is no default
 read_only for a mutating tool to inherit by forgetting.
 A tool's docstring IS its contract with the agent — it ships as the tool's
 description. Say what it is for, when to reach for it over its siblings, what
 each argument's empty value means, and what comes back.
 """
 from __future__ import annotations
 from . import alerts, audit, automation, hosts, metrics, monitors, overview, session
 _MODULES = (session, overview, hosts, monitors, alerts, metrics, audit, automation)
 def register_all(register_tool) -> None:
    for module in _MODULES:
        module.register(register_tool)
@@ -1,24 +0,0 @@
 """Audit trail for changes made through MCP.
 Every mutating tool records its change with `audit_mcp`, under the calling
 user's id and the username "<user> via MCP" (an operator ruling on the API
 Surface), so the audit log separates what a person did from what an agent
 did on their token. The token id goes into the detail.
 """
 from __future__ import annotations
 from steward.core.audit import log_audit
 from steward.mcp._context import current_caller, mcp_app
 VIA = " via MCP"
 def mcp_username(username: str) -> str:
    return (username + VIA)[:64]
 async def audit_mcp(action: str, entity_type: str | None = None, entity_id: str | None = None,
                    detail: dict | None = None) -> None:
    caller = current_caller()
    await log_audit(mcp_app(), caller.user_id, mcp_username(caller.username), action,
                    entity_type, entity_id, {**(detail or {}), "token_id": caller.token_id})
@@ -1,149 +0,0 @@
 """Response shaping every tool shares: the conventions INSTRUCTIONS promise.
 - Times are ISO-8601 UTC with a `Z`, and anything with a "when" also carries
  `age_s` so the agent never does clock arithmetic.
 - Every list is bounded: a tool returns `total`, `truncated`, and when cut, a
  `more` line naming the argument that fetches the rest. Nothing is cut
  silently.
 - A time series comes back as a summary plus a downsampled series, never raw
  rows.
 - An entity is looked up by id or by name, and a miss raises ValueError naming
  the tool that lists the valid choices (the server turns it into a readable
  ToolError).
 """
 from __future__ import annotations
 import difflib
 import math
 from datetime import datetime, timezone
 from typing import Callable, Iterable, TypeVar
 from urllib.parse import urlsplit, urlunsplit
 from steward.core.time_range import RANGES
 from steward.mcp._context import mcp_app
 T = TypeVar("T")
 SERIES_POINTS = 60  # max points in any returned series
 def db_session():
    """A session on the calling Steward's database; use as `async with db_session() as db`."""
    return mcp_app().db_sessionmaker()
 def now_utc() -> datetime:
    return datetime.now(timezone.utc)
 def _aware(dt: datetime) -> datetime:
    return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
 def iso(dt: datetime | None) -> str | None:
    if dt is None:
        return None
    return _aware(dt).astimezone(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
 def age_s(dt: datetime | None, now: datetime | None = None) -> int | None:
    if dt is None:
        return None
    return max(0, int(((now or now_utc()) - _aware(dt)).total_seconds()))
 def rnd(value, digits: int = 2):
    return None if value is None else round(float(value), digits)
 def clamp_limit(limit: int, default: int, maximum: int) -> int:
    """`limit` <= 0 means the default; anything above `maximum` is capped."""
    if not limit or limit <= 0:
        return default
    return min(int(limit), maximum)
 def bounded(items: list, limit: int, *, key: str = "items", more: str = "") -> dict:
    """`{key: items[:limit], total, truncated}`, plus `more` when something was cut."""
    total = len(items)
    out = {key: items[:limit], "total": total, "truncated": total > limit}
    if total > limit:
        out["more"] = f"Showing {limit} of {total}. " + (more or "Pass a larger `limit`.")
    return out
 def lookback(range_: str) -> tuple[datetime, str]:
    """A lookback window as (since, key). Unlike core parse_range, an unknown key
    is refused rather than quietly read as 24h: the agent should learn its typo."""
    key = (range_ or "24h").strip()
    if key not in RANGES:
        raise ValueError(f"range {range_!r} is not one of {', '.join(RANGES)}.")
    return now_utc() - RANGES[key], key
 def downsample(points: list[tuple[datetime, float]], n: int = SERIES_POINTS) -> list[tuple[datetime, float]]:
    """Average consecutive points into at most n, each stamped with its first time."""
    if len(points) <= n:
        return points
    size = math.ceil(len(points) / n)
    out = []
    for i in range(0, len(points), size):
        chunk = points[i:i + size]
        out.append((chunk[0][0], sum(v for _, v in chunk) / len(chunk)))
    return out
 def summarize(points: list[tuple[datetime, float]], n: int = SERIES_POINTS) -> dict:
    """min/avg/max/p95/last over oldest-first (time, value) points, plus a ≤n series."""
    if not points:
        return {"points": 0, "series": []}
    values = [v for _, v in points]
    ordered = sorted(values)
    p95 = ordered[max(0, math.ceil(0.95 * len(ordered)) - 1)]
    return {
        "points": len(values),
        "min": rnd(ordered[0]),
        "avg": rnd(sum(values) / len(values)),
        "max": rnd(ordered[-1]),
        "p95": rnd(p95),
        "last": rnd(values[-1]),
        "last_at": iso(points[-1][0]),
        "series": [[iso(t), rnd(v)] for t, v in downsample(points, n)],
    }
 def pick(want: str, rows: Iterable[T], *, kind: str, list_tool: str,
         name: Callable[[T], str] = lambda r: r.name,
         ident: Callable[[T], str] = lambda r: r.id) -> T:
    """The row whose id is `want`, else the one whose name matches (case-insensitive)."""
    rows = list(rows)
    want = (want or "").strip()
    if not want:
        raise ValueError(f"pass a {kind} name or id; {list_tool} lists them.")
    for r in rows:
        if ident(r) == want:
            return r
    same = [r for r in rows if (name(r) or "").lower() == want.lower()]
    if len(same) == 1:
        return same[0]
    if same:
        ids = ", ".join(f"{ident(r)}" for r in same)
        raise ValueError(f"{len(same)} {kind}s are named {want!r}; pass one of these ids instead: {ids}.")
    close = difflib.get_close_matches(want.lower(), [(name(r) or "").lower() for r in rows], n=3, cutoff=0.5)
    hint = f" Did you mean {', '.join(repr(c) for c in close)}?" if close else ""
    raise ValueError(f"no {kind} named or with id {want!r}.{hint} {list_tool} lists them.")
 def redact_url(url: str) -> str:
    """Drop a password from a URL's userinfo; the rest stays readable."""
    try:
        parts = urlsplit(url)
    except ValueError:
        return url
    if parts.password is None:
        return url
    host = parts.hostname or ""
    if parts.port:
        host = f"{host}:{parts.port}"
    netloc = f"{parts.username}:***@{host}" if parts.username else f"***@{host}"
    return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
--- a/Show More
+++ b/Show More