bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Compare commits

base: bvandeusen/FabledSteward:ea31deabf93da88eaf6f584622d4e18e6601e198
Branches Tags
bvandeusen/FabledSteward:main bvandeusen/FabledSteward:dev bvandeusen/FabledSteward:renovate/configure bvandeusen/FabledSteward:feat/host-agent
..
compare: bvandeusen/FabledSteward:renovate/configure
Branches Tags
bvandeusen/FabledSteward:dev bvandeusen/FabledSteward:renovate/configure bvandeusen/FabledSteward:main bvandeusen/FabledSteward:feat/host-agent
2 Commits
ea31deabf9 .. renovate/configure
Author SHA1 Message Date
Renovate Bot 548a3f711a Add renovate.json 2026-08-04 04:01:54 +00:00
bvandeusen 001cbafe56 Merge pull request 'Ansible schedule form: structured playbook-variable fields + first-item dropdown defaults' (#2) from dev into main
CI / lint (push) Successful in 2s
Details
CI / unit (push) Successful in 46s
Details
CI / integration (push) Successful in 2m21s
Details
CI / publish (push) Successful in 4s
Details
2026-06-30 23:44:04 -04:00
475 changed files with 7670 additions and 38933 deletions
Expand all files Collapse all files
.forgejo/workflows/ci.yml
+8 -132
View File
@@ -14,17 +14,6 @@ on:
# pull_request intentionally absent — push on [dev, main] already fires CI for
# every dev commit and dev→main PRs. Single-operator repo, no fork PRs.
# Serialize CI runs per ref so the publish lane never shares the runner's docker
# daemon with another run. Two publishes racing on one daemon evict each other's
# freshly-built image mid-push, breaking the post-build tag/push (issue #1093).
# cancel-in-progress:false is deliberate — rule 46 requires EVERY dev/main push to
# publish its own immutable :<sha> image, so a superseded run must still run to
# completion (never cancelled) to emit its SHA. (Forgejo honors `concurrency:` —
# CI-runner's build workflows use it.)
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: false
jobs:
# Fast-fail lint lane. ruff is pre-installed in the ci-python image, so this
# runs with NO dependency install and surfaces lint bounces in seconds.
@@ -80,24 +69,14 @@ jobs:
# act_runner (swarm-runner) puts service containers on the default bridge
# with no embedded DNS, so the hostname `postgres` won't resolve — we
# discover the service container's bridge IP via the docker socket. The
# job container and its Postgres share a task-number name prefix.
# job name `integration` (no underscore) is the docker-ps name filter.
run: |
set -euxo pipefail
echo "=== container landscape (diagnostic for filter scoping) ==="
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
echo "=== end landscape ==="
# The runner's daemon is shared: another repo's integration job can be
# running beside this one, and `name=integration` matches its Postgres
# too (run 8358 took the wrong one and every test failed to connect).
# Scope to this job's own task: act_runner names every container of a
# task GITEA-ACTIONS-TASK-<n>-…, so read <n> from our own container
# (its id is in the /etc/hostname bind mount) and require one match.
SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
SELF=${SELF:-$(hostname)}
TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
test -n "$TASK"
PG=$(docker ps --filter "name=$TASK" --filter "ancestor=postgres:16-alpine" -q)
test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
PG=$(docker ps --filter "name=integration" --filter "ancestor=postgres:16-alpine" -q | head -n1)
test -n "$PG"
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
test -n "$PG_IP"
export STEWARD_DATABASE_URL="postgresql+asyncpg://steward:steward@$PG_IP:5432/steward"
@@ -132,122 +111,19 @@ jobs:
# 401s with it. REGISTRY_TOKEN is a repo Actions secret holding a
# Forgejo token scoped read:package + write:package.
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.fabledsword.com -u bvandeusen --password-stdin
- name: Build (commit-SHA always; :dev on dev, :latest on main)
- name: Build + push (commit-SHA always; :dev on dev, :latest on main)
run: |
set -eux
IMAGE=git.fabledsword.com/bvandeusen/steward
# The moving tag for this ref: dev→:dev, main→:latest (rule 46). main IS
# the production line, so :latest tracks main's tip; there is no :main.
MOVING=""
if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
MOVING="dev"
elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
MOVING="latest"
fi
# Tag BOTH targets at build time (mirrors CI-runner's build workflows):
# one `docker build -t :<sha> -t :<moving>` points both tags at the image
# atomically, so we never run a post-push `docker tag` of an image a
# concurrent run could have evicted from the shared daemon (issue #1093).
if [ -n "$MOVING" ]; then
docker build -t "$IMAGE:${{ github.sha }}" -t "$IMAGE:$MOVING" .
else
docker build -t "$IMAGE:${{ github.sha }}" .
fi
# Rule 164's check, BETWEEN build and push (rule 177): an image that
# can't boot, migrate and serve with no internet never reaches a tag.
# An --internal network lets it reach its own Postgres and nothing else;
# --network none can't work for an app that needs a database to start.
# Embedded DNS works on a user-defined network, unlike the services:
# containers above (rule 79), so the app finds Postgres by name.
- name: The image boots with no internet
env:
SMOKE: steward-smoke-${{ github.run_id }}
run: |
set -eux
IMAGE=git.fabledsword.com/bvandeusen/steward:${{ github.sha }}
docker network create --internal "$SMOKE-net"
docker run -d --name "$SMOKE-pg" --network "$SMOKE-net" \
-e POSTGRES_USER=steward -e POSTGRES_PASSWORD=steward -e POSTGRES_DB=steward \
postgres:16-alpine
# Waits with deadlines (rule 156); pg_isready, not /dev/tcp, which is
# a bash builtin and never succeeds under sh.
ready=""
for _ in $(seq 1 60); do
if docker exec "$SMOKE-pg" pg_isready -U steward -d steward -q; then ready=1; break; fi
sleep 2
done
test -n "$ready" || { echo "FATAL: postgres not ready within 120s"; exit 1; }
docker run -d --name "$SMOKE-app" --network "$SMOKE-net" \
-e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/steward" \
-e STEWARD_SECRET_KEY=smoke-test-placeholder \
"$IMAGE"
# The image has no curl; its own Python fetches. Prints the body.
fetch() {
docker exec "$SMOKE-app" python -c 'import sys, urllib.request
r = urllib.request.urlopen("http://127.0.0.1:5000" + sys.argv[1], timeout=5)
sys.stdout.write(r.read().decode())' "$1"
}
# Migrations run before the server starts, so /health answering
# means the schema applied against a real database.
up=""
for _ in $(seq 1 60); do
if fetch /health >/dev/null 2>&1; then up=1; break; fi
sleep 2
done
if [ -z "$up" ]; then
echo "FATAL: Steward never answered /health with no internet access"
docker logs "$SMOKE-app" 2>&1 | tail -80
exit 1
fi
fetch /health | grep -q '"ok"'
# tini is PID 1, so zombies are reaped and SIGTERM reaches Steward.
test "$(docker exec "$SMOKE-app" cat /proc/1/comm)" = "tini"
# The zone database is in the image, so a reader's zone resolves
# instead of quietly falling back to UTC (milestone 490).
docker exec "$SMOKE-app" python -c 'from zoneinfo import ZoneInfo; ZoneInfo("America/New_York")'
# A fresh database lands on first-run setup: a real page, its assets
# served by Steward itself, and no third-party host in it (rule 155).
fetch / > /tmp/smoke-page.html
grep -qi "<!doctype html" /tmp/smoke-page.html
grep -q "/static/" /tmp/smoke-page.html
if grep -Eqi 'https?://(fonts\.|cdn\.|unpkg\.|cdnjs\.|ajax\.)' /tmp/smoke-page.html; then
echo "FATAL: the page references a third-party host"
grep -Eoi 'https?://[^"'"'"' ]+' /tmp/smoke-page.html
exit 1
fi
# A stop is a clean shutdown: SIGTERM through tini to Hypercorn, which
# finishes within its graceful timeout and exits 0.
docker stop -t 30 "$SMOKE-app"
code="$(docker inspect -f '{{.State.ExitCode}}' "$SMOKE-app")"
test "$code" = "0" || { echo "FATAL: Steward exited $code on SIGTERM"; exit 1; }
- name: Push
run: |
set -eux
set -euxo pipefail
IMAGE=git.fabledsword.com/bvandeusen/steward
docker build -t "$IMAGE:${{ github.sha }}" .
docker push "$IMAGE:${{ github.sha }}"
if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:dev"
docker push "$IMAGE:dev"
elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:latest"
docker push "$IMAGE:latest"
fi
- name: Smoke logs
if: always()
run: docker logs "steward-smoke-${{ github.run_id }}-app" 2>&1 | tail -60 || true
# A failed smoke must not leave a Postgres and an app on the shared daemon.
- name: Smoke clean-up
if: always()
run: |
docker rm -f "steward-smoke-${{ github.run_id }}-app" "steward-smoke-${{ github.run_id }}-pg" >/dev/null 2>&1 || true
docker network rm "steward-smoke-${{ github.run_id }}-net" >/dev/null 2>&1 || true
- name: Prune dangling layers
if: always()
run: docker image prune -f
Dockerfile
+7 -17
View File
@@ -1,5 +1,4 @@
# The Python CI tests on (ci-python:3.14), so the image runs what was tested.
FROM python:3.14-slim
FROM python:3.13-slim
RUN DEBIAN_FRONTEND=noninteractive apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
@@ -11,14 +10,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \
# sshpass — Ansible shells out to it for first-contact password SSH
# (provisioning a fresh host before the managed key is installed)
sshpass \
# git — clones and pulls Ansible git sources (Settings > Ansible)
git \
# tini — PID 1: reaps the zombies ssh and ansible leave behind and
# forwards SIGTERM to Steward so a stop is a graceful shutdown
tini \
# tzdata — the zone database, so times show in each reader's own zone
# (milestone 490); Debian's slim base no longer carries it
tzdata \
&& rm -rf /var/lib/apt/lists/*
# Upgrade pip to suppress the version notice
@@ -28,11 +19,10 @@ WORKDIR /app
COPY pyproject.toml .
COPY steward/ steward/
# Bundle the extras whose features ship in the image: [ansible] for the
# playbook runner, [snmp] (pysnmp) for the SNMP poller, [ldap] (ldap3, pure
# Python) for LDAP sign-in under Settings -> Auth. Without them those features
# load but fail for want of their runtime dependency.
RUN pip install --no-cache-dir '.[ansible,snmp,ldap]'
# Bundle the extras whose first-party plugins ship in the image: [ansible] for
# the playbook runner, [snmp] (pysnmp) for the SNMP poller. Without them those
# bundled plugins load but silently no-op for want of their runtime dependency.
RUN pip install --no-cache-dir '.[ansible,snmp]'
COPY alembic.ini .
# First-party plugins ship inside the image (bundled root at /app/plugins).
@@ -46,10 +36,10 @@ RUN chmod +x /entrypoint.sh
# so the entrypoint can fix up /data permissions if needed, then drops to
# 'app' (uid 1000) via gosu before launching steward.
RUN useradd -m -u 1000 app \
&& mkdir -p /data/ansible/sources /data/ansible/runs /data/ansible/playbooks /data/plugins \
&& mkdir -p /data/playbook_cache /data/plugins \
&& chown -R app:app /data /app
EXPOSE 5000
ENTRYPOINT ["/usr/bin/tini", "--", "/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["steward", "--host", "0.0.0.0", "--port", "5000"]
ci-requirements.md
+1 -1
View File
@@ -46,7 +46,7 @@ git.fabledsword.com/bvandeusen/ci-python:3.14
- Steward has **no frontend and no Redis/Celery** (no external workers), so there
is no frontend-build lane and the only service container is Postgres.
- Integration uses Fabled-Git Actions `services:` + a socket-discovered bridge IP
- Integration uses Forgejo Actions `services:` + a socket-discovered bridge IP
because `act_runner` (swarm-runner v0.6+) puts services on the default bridge
with no embedded DNS. FabledCurator's `ci.yml` is the canonical example of the
pattern; Steward's is the same shape minus Redis and minus sharding.
config.example.yaml
-8
View File
@@ -13,14 +13,6 @@
database:
url: "postgresql+asyncpg://steward:password@localhost/steward"
# Optional: seconds to wait at startup for the database to become reachable
# before giving up (default: 60). Steward retries with backoff rather than
# crashing when the DB is merely slow to arrive — replaying WAL after an
# unclean shutdown, or container DNS not yet resolving after a host reboot.
# Raise it if your database is routinely slower than this to accept
# connections. Env var: STEWARD_DB_CONNECT_TIMEOUT
# connect_timeout: 60
# Optional: override the auto-generated secret key.
# If not set, a key is auto-generated on first run and saved to /data/secret.key.
# secret_key: "change-me-to-a-random-string"
docs/architecture.md
-24
View File
@@ -44,30 +44,6 @@ All routes are Quart Blueprints registered in `app.py`:
Plugin blueprints are mounted automatically by `load_plugins()` using the plugin directory name as the URL prefix.
`/ai-access/` (`ai_access_bp`, `steward/ai_access/routes.py`) is where a signed-in user manages their API tokens.
### `/mcp`: outside the blueprint pipeline
`mount_mcp(app)` (`steward/mcp/server.py`) wraps `app.asgi_app`. A request to
`/mcp` is handed to the MCP SDK's streamable-HTTP app before Quart sees it, so
no blueprint, session cookie or `before_request` hook runs for it. Each
request then goes through these steps:
1. **Database gate.** If the database is unreachable, the request gets a 503
with a readable message.
2. **Bearer resolution.** `api_tokens.resolve_token()` turns the token into a
`Caller` (user, effective role, token id). A missing, revoked or expired
token gets a 401.
3. **Caller context.** The `Caller` and the app are set in ContextVars
(`steward/mcp/_context.py`). Tools read them through `current_caller()` and
`mcp_app()`, because Quart's `current_app` and `session` do not exist there.
4. **Role gate.** `StewardMCPServer.refusal()` hides and refuses tools above
the caller's role, and plugin tools whose plugin is disabled.
The SDK's session manager runs in Quart's `before_serving`/`after_serving`
lifecycle. The server is stateless: no MCP session outlives a request. See
[MCP server](core/mcp.md) for the security model and the tools.
---
## Scheduler
docs/core/alerting.md
+1 -2
View File
@@ -8,7 +8,7 @@ Alert rules evaluate every metric that flows through `record_metric()`. There is
When any monitor or plugin calls `record_metric()`:
1. The metric value is written to the `plugin_metrics` table, and `plugin_metric_latest` moves to it if it is the newest sample of that metric
1. The metric value is written to the `plugin_metrics` table
2. All enabled `AlertRule` rows matching `(source_module, resource_name, metric_name)` are loaded
3. Each matching rule is evaluated against the new value
4. If a state transition occurs, an `AlertEvent` row is written
@@ -136,4 +136,3 @@ Defined in `steward/models/alerts.py`:
- **`alert_states`** — one row per rule, tracks current state and consecutive failure count
- **`alert_events`** — append-only log of all state transitions and notification outcomes
- **`plugin_metrics`** — all metric values written by any monitor or plugin
- **`plugin_metric_latest`** — the newest sample of each (source, resource, metric). Pages that show "what it is now" read this instead of searching history; retention drops a row once its sample leaves the raw window
docs/core/ansible.md
+3 -10
View File
@@ -31,9 +31,10 @@ ansible.sources:
url: "https://github.com/example/infra.git"
branch: "main"
pull_interval_seconds: 300
cache_path: "/data/playbook_cache/infra-repo"
```
Each git source is checked out under `/data/ansible/sources/<name>` (override with `STEWARD_ANSIBLE_SOURCE_DIR`), on the persistent volume, so a redeploy keeps it. One scheduled task, `ansible_git_pulls`, ticks every 30 s and pulls each source once its `pull_interval_seconds` has passed; a source added in Settings is pulled within a tick, with no restart. A clone or pull that runs past 120 s is cut (its process group terminated), a clone that fails is removed so the next attempt starts clean, and git never prompts for credentials.
Git sources register a `ScheduledTask` named `ansible_git_pull_<name>` that runs `git pull` on the configured interval.
---
@@ -101,20 +102,12 @@ If a client connects after the run has already completed, the endpoint immediate
| Status | Description |
|---|---|
| `queued` | Waiting for a host another run holds, or for a run slot |
| `running` | Execution in progress |
| `success` | Playbook exited 0 |
| `failed` | Playbook exited non-zero |
| `interrupted` | App restarted while run was in progress |
| `cancelled` | Stopped by an operator, or by Steward shutting down mid-run |
On startup, the app marks any runs still `running` or `queued` as `interrupted` (see `_mark_interrupted_runs()` in `app.py`).
### Overlapping runs
- **One run per host.** Before it takes a run slot, a run claims every host it touches. Hosts come from the inventory Steward writes, keyed by address. A run on a source's own inventory file claims that file, because Steward doesn't read its hosts. While another run holds one of its hosts, the run shows `queued` and says which run it waits for. After an hour of waiting it fails and says so.
- **One run at a time per schedule.** A schedule that falls due while its previous run is still `queued` or `running` skips that firing. The Schedules page shows "Skipped: the previous run was still going" beside that run. The next firing comes an interval later.
- **One action run at a time per alert rule.** When a rule fires again while the action run from an earlier firing is still going, no new run starts. The firing's event says it was skipped and names the run.
On startup, the app marks any runs still in `running` state as `interrupted` (see `_mark_interrupted_runs()` in `app.py`).
Output stored in the DB is capped at 1 MB. If truncated, `[output truncated]` is appended to the DB column, but the live SSE stream continues unaffected.
docs/core/mcp.md
-132
View File
@@ -1,132 +0,0 @@
# MCP server
Steward serves the [Model Context Protocol](https://modelcontextprotocol.io) at
`/mcp`. An AI client connected to it can look into everything Steward watches
(hosts, monitors, alerts, metrics, containers, playbook runs) and, with a
high enough role, change it. A token's role sets that ceiling.
**Source:** `steward/mcp/`. The tools live in `steward/mcp/tools/`, and a
plugin's tools live in `plugins/<name>/mcp_tools.py`.
**Transport:** streamable HTTP, stateless. Every request carries its own
token, so a client survives a Steward restart without reconnecting.
---
## Connecting
1. In Steward, open **AI access** and create a token. Pick the lowest role
that does the job.
2. Copy the token (it is shown once) and use one of the recipes on that
page. They carry the real endpoint URL and the new token. For Claude Code:
```sh
claude mcp add --transport http steward https://steward.example/mcp \
--header "Authorization: Bearer stw_…"
```
3. Ask the client to call `whoami`. It answers with the user, the effective
role and what that role allows.
Claude Desktop connects through `mcp-remote`. Open WebUI and other clients
need an MCP server with the **Streamable HTTP** transport and bearer
authentication. The AI access page shows the recipes for both.
### Behind a reverse proxy
- The URL on the AI access page is **Settings → General → Public base URL**
plus `/mcp`. If that setting is empty, the URL is built from the request's
host instead.
- The proxy must pass the `Authorization` header through unchanged.
- It must not buffer responses. Replies may arrive as server-sent events
(`text/event-stream`). For nginx, set `proxy_buffering off;` on the
location.
- Steward does no Host-header (DNS-rebinding) check on `/mcp`, because the
bearer token is the security boundary. Serve it over HTTPS.
---
## Security model
| Rule | How it is enforced |
|---|---|
| Bearer tokens only, no OAuth | `steward/core/api_tokens.py`. Only a hash is stored, and the token is shown once. |
| A token never acts above its user | Effective role = min(token role, user's **current** role). Lowering a user lowers every token they hold; disabling the user stops them all. |
| Default deny | A tool is reachable only if it was registered with a role (`register_tool(fn, role=..., read_only=...)`). `tools/list` shows a caller only the tools their role allows. A call above the role is refused, and the refusal names the role it needs. |
| Plugin tools follow the plugin | A disabled plugin's tools are hidden and refused. |
| Dry run first | `run_playbook`, `create_schedule` and `set_alert_action` default to Ansible check mode (`--check --diff`). Changes are applied only when the caller passes `check=false`. A playbook its author flagged `# steward:confirm: true` also needs `confirmed=true`. `prune_docker` previews unless `check=false`. |
| Secrets stay put | Playbook variables Steward detects as secret reach the run but are never stored, audited or returned. Schedules and alert actions refuse them, since they would have to store them. Inventory variables are listed by name only, and HTTP monitor header values are redacted. |
| Every change is audited | Each write records an audit event as `<user> via MCP`, with the token id in its detail. `list_audit_events(user="<user>")` includes those events. |
| Unknown arguments are refused | A misspelt argument is rejected with a "did you mean", rather than silently dropped. |
Tokens never mint or revoke tokens; that happens only on the AI access
page, in a browser session.
---
## Tools by role
Each role includes the tools of the roles above it in this list.
### Viewer: read everything
| Area | Tools |
|---|---|
| Start here | `whoami`, `get_overview` (what is wrong now, ranked, each finding with the call that shows its evidence) |
| Hosts and metrics | `list_hosts`, `get_host`, `get_host_metrics`, `query_metrics` |
| Monitors | `list_monitors`, `get_monitor` |
| Alerts | `list_alerts`, `get_alert_history`, `list_alert_rules`, `get_alert_rule_options`, `list_maintenance_windows` |
| Automation | `list_playbooks`, `get_playbook`, `list_inventory`, `list_ansible_runs`, `get_ansible_run`, `wait_for_run`, `list_schedules` |
| Audit | `list_audit_events` |
| Docker plugin | `list_containers`, `get_container`, `list_swarm_services`, `search_container_logs`, `get_docker_disk_usage` |
| Host agent plugin | `list_agents` |
### Operator: run playbooks, change monitoring
| Area | Tools |
|---|---|
| Automation | `run_playbook`, `cancel_ansible_run`, `create_schedule`, `update_schedule`, `delete_schedule` |
| Monitors | `create_monitor`, `update_monitor`, `set_monitor_enabled` |
| Alerts | `create_alert_rule`, `update_alert_rule`, `acknowledge_alert`, `create_maintenance_window`, `end_maintenance_window` |
### Admin: destructive and remediation
`set_alert_action` (the playbook a rule runs when it fires), `delete_monitor`,
`delete_alert_rule`, `prune_docker`.
The client sees each tool's full contract: arguments, what an empty value
means, and what comes back. That contract is the tool's docstring, so read the
docstrings in `steward/mcp/tools/` for the detail.
---
## Response conventions
Every tool follows these conventions (`steward/mcp/tools/_shape.py`):
- **Times** are ISO-8601 UTC with a `Z`. Anything with a "when" also carries
`age_s`, so the client never does clock arithmetic.
- **Lists are bounded.** A list response carries `total` and `truncated`. When
it was cut, `more` names the argument that fetches the rest. Nothing is cut
silently.
- **Time series** come back as a summary (min/avg/max/p95/last) plus a series
of at most 60 points, never raw rows.
- **Lookup by name or id.** A miss suggests close matches and names the tool
that lists the valid choices.
- **Refusals are readable.** Bad input becomes a tool error that says what was
wrong, why, and what to do instead, and nothing is changed. The tools share
their validation with the web forms (`monitors/service.py`,
`alerts/service.py`), so a value the UI refuses is refused here for the same
reason.
- **Database outages** return a plain "Steward cannot reach its database"
rather than a stack trace.
---
## Adding tools
Core tools: add an `async def` to a module in `steward/mcp/tools/` and list it
in that module's `register()` with its minimum role and `read_only`. A plugin
exposes `register_mcp_tools(register)`, and optionally `mcp_overview(db)` to
contribute findings to `get_overview`. See
[Writing a plugin](../plugins/writing-a-plugin.md). A tool that changes
anything calls `audit_mcp(...)` from `steward/mcp/tools/_audit.py`.
docs/plugins/host-agent-design.md
+3 -17
View File
@@ -74,15 +74,8 @@ token = a1b2c3d4...
interval_seconds = 30
hostname = myhost # optional; defaults to uname -n
mounts = /, /mnt/data # optional; defaults to all real mounts (excluding tmpfs/devtmpfs/etc.)
docker_logs_enabled = true # optional; on by default — per-host container-log kill-switch
docker_log_exclude = watchtower, noisy-svc # optional; container names whose logs this host skips collecting
```
Container-log collection is on by default (no config needed). The two keys above
are the per-host opt-outs; the operator-facing global toggle + exclude live in
Settings and are enforced server-side (the push model has no channel to command
an agent), so they don't require touching a host's conf.
### Agent internals (function list, not classes)
- `read_config(path)` — parses the conf file into a dict.
@@ -92,17 +85,10 @@ an agent), so they don't require touching a host's conf.
- `collect_load()` — reads `/proc/loadavg`, returns `[1m, 5m, 15m]`.
- `collect_uptime()` — reads `/proc/uptime`, returns seconds since boot (int).
- `collect_metadata()` — `os.uname()` for kernel + arch, `/etc/os-release` for distro. Called once at startup and cached.
- `collect_docker_logs(socket, containers, state, exclude)` — (m79) per running
container, fetches new log lines over the Docker socket with an incremental
`since` cursor kept per container in `state` (a container's first interval
seeds from a short tail). Demuxes the multiplexed stream, parses each line's
RFC3339 timestamp, caps the whole batch at a byte limit (a marker line records
a truncation; the deferred lines come next interval). Folded into the sample as
`docker_logs`; omitted when empty.
- `build_payload()` — assembles a snapshot from all collectors into one dict.
- `post_payload(url, token, payloads)` — POSTs a list of samples, returns success/failure.
- `RingBuffer(maxlen=20)` — tiny FIFO wrapper, drops oldest when full.
- `main_loop()` — the 30s loop: collect → try POST → on failure push to buffer + backoff → on success flush buffer. Container logs are stripped from a sample before it's buffered (metrics survive an outage; stale logs are dropped).
- `main_loop()` — the 30s loop: collect → try POST → on failure push to buffer + backoff → on success flush buffer.
**Target: ~300 lines total including docstrings.** More than that is a smell that the agent is over-scoping.
@@ -157,6 +143,7 @@ plugins/host_agent/
│ └── host_agent_001_initial.py # creates host_agent_registrations table
├── templates/
│ ├── detail.html # per-host detail page
│ ├── widget_table.html # dashboard fleet table widget
│ ├── widget_history.html # dashboard history chart widget
│ ├── settings_list.html # plugin settings: list of registered hosts
│ └── install.sh.j2 # install script template (Jinja)
@@ -268,7 +255,6 @@ Content-Type: application/json
- **`metadata` is sent on every POST**, not just on change. Server-side diff detects actual changes and only writes on change. Cost per POST is one dict — negligible. Benefit: server can cleanly detect agent restarts.
- **Raw bytes, not percentages, for memory and storage.** Percentages are derived server-side. Changing the "what counts as used" math doesn't require re-releasing the agent.
- **CPU is the one exception** — reported as a percentage because it's inherently a derivative (delta over time), not a snapshot. The agent must sample twice to compute it.
- **Container logs (m79) ride in the same push** as `docker_logs`: a list of `{container, stream, ts, line}` records — incremental since the previous interval. This pushes the *same direction* as metrics, so batched log history needs no inbound channel (only sub-second live-follow would). The server ingests them into `docker_logs`, enforces the global toggle + exclude list on ingest, and bounds storage with a per-container size+age ring. `docker_logs` is omitted when there's nothing new.
### Server expansion into `PluginMetric` rows
@@ -437,7 +423,7 @@ echo "Logs: journalctl -u steward-agent -f"
### Dashboard widgets (`steward/core/widgets.py`)
- The fleet glance (CPU, mem, disk /, load 1m, stale mark) is the core **Hosts — Overview** widget (`hosts_overview`); the old `host_resources` table was folded into it (#5213).
- **`host_resources`** — table widget. One row per monitored host: name, CPU %, mem %, disk % (worst mount), load 1m, "last seen Xs ago" with red/yellow/green coloring. Fleet glance.
- **`host_resource_history`** — chart widget for one host. CPU / mem / disk over a selectable time range (1h, 6h, 24h, 7d). Same pattern as every other history widget — Chart.js.
### Per-host detail page
docs/plugins/index.yaml.example
+113
View File
@@ -0,0 +1,113 @@
# steward-plugins / index.yaml
#
# This file is the catalog index for the steward plugin repository.
# It is fetched by the app's Settings → Plugins → Plugin Catalog UI.
#
# Steward reads this file from:
# https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml
#
# After adding or updating a plugin entry, commit and push — the change is
# live immediately for anyone whose app fetches the catalog (cache TTL: 5 min).
#
# REQUIRED fields for each plugin entry:
# name - must match the plugin's directory name exactly
# version - semver string
# download_url - direct URL to a zip file containing the plugin
# checksum_sha256 - SHA-256 hex digest of the zip (leave empty to skip verification)
#
# OPTIONAL fields:
# description, author, license, min_app_version,
# repository_url, homepage, tags
#
# Generating a checksum:
# sha256sum traefik.zip
# shasum -a 256 traefik.zip (macOS)
#
# Download URL conventions:
# Gitea release assets (recommended):
# https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip
# Upload the zip as a release attachment in Gitea; paste the URL here.
# Source archive tarballs work too but release assets are preferred (smaller, plugin-only).
version: 1
updated: "2026-03-22"
plugins:
- name: http
version: "1.0.0"
description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/http"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/http-v1.0.0/http.zip"
checksum_sha256: ""
tags:
- monitoring
- http
- synthetic
- uptime
- name: docker
version: "1.0.0"
description: "Docker container status, resource usage, and restart tracking via Docker socket"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v1.0.0/docker.zip"
checksum_sha256: ""
tags:
- containers
- docker
- infrastructure
- name: traefik
version: "1.0.0"
description: "Traefik reverse proxy metrics and access log integration"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip"
checksum_sha256: "" # fill in after running: sha256sum traefik.zip
tags:
- proxy
- metrics
- access-log
- name: unifi
version: "1.0.0"
description: "UniFi Network controller integration — WAN health, devices, clients, DPI"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/unifi-v1.0.0/unifi.zip"
checksum_sha256: ""
tags:
- network
- unifi
- ubiquiti
- name: host_agent
version: "1.0.0"
description: "Remote Linux host resource monitoring via a lightweight Python push agent"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/host_agent"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/host_agent-v1.0.0/host_agent.zip"
checksum_sha256: ""
tags:
- host
- monitoring
- cpu
- memory
- storage
docs/plugins/writing-a-plugin.md
+60 -59
View File
@@ -28,7 +28,7 @@ license: "MIT" # any SPDX identifier, e.g. MIT, Apache-2.0, GPL-3.0
# Optional: prevents loading on older app versions
min_app_version: "0.1.0"
# Optional: where the plugin's source lives
# Optional: shown in the catalog UI
repository_url: "https://github.com/yourname/yourrepo"
homepage: "https://github.com/yourname/yourrepo/tree/main/myplugin"
tags:
@@ -323,82 +323,83 @@ Role hierarchy: `admin > operator > viewer`. Requiring `viewer` grants access to
---
## Audit Log Entries
## Publishing to the Catalog
A change made through a plugin's pages is audited with `steward.core.audit.log_audit`, naming the thing it changed by `entity_type` and `entity_id`. So the audit page shows that thing's current name, linked to its page, export `get_audit_entities()` from `__init__.py`:
The official plugin catalog is hosted at `https://git.fabledsword.com/bvandeusen/Steward-plugins`. Anyone can submit a plugin by opening a pull request — first-party and third-party plugins are treated identically by the catalog system.
```python
# plugins/myplugin/__init__.py
def get_audit_entities() -> dict:
from .models import Target
return {"myplugin_target": (Target, "/plugins/myplugin/targets/{}")}
### Repo layout
```
steward-plugins/
├── index.yaml ← catalog index — the only file Steward fetches
├── myplugin/
│ ├── plugin.yaml
│ ├── __init__.py
│ └── ...
└── .github/
└── workflows/
└── publish.yml ← packages each plugin dir into a zip on release
```
The model needs `id` and `name` columns. An entity deleted since keeps the name the entry recorded in its detail (`detail={"name": ...}`). Add the action's wording to `ACTIONS` in `steward/audit/service.py` so the row reads as a sentence.
### index.yaml entry
---
Each plugin needs a corresponding entry in `index.yaml`. See `docs/plugins/index.yaml.example` for the full schema. The minimum required fields are:
## Agent Tools (MCP)
Steward serves an MCP endpoint at `/mcp` that AI clients reach with a personal API token created on the AI access page. A plugin adds tools to it by exporting an optional `register_mcp_tools(register)` from `__init__.py`:
```python
# plugins/myplugin/__init__.py
def register_mcp_tools(register) -> None:
from .mcp_tools import list_targets, rescan_target
from steward.models.users import UserRole
register(list_targets, role=UserRole.viewer, read_only=True)
register(rescan_target, role=UserRole.operator, read_only=False)
```yaml
- name: myplugin
version: "1.0.0"
description: "What this plugin does"
author: "Your Name"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://github.com/yourname/yourrepo"
homepage: "https://github.com/yourname/yourrepo/tree/main/myplugin"
download_url: "https://github.com/yourname/yourrepo/releases/download/myplugin-v1.0.0/myplugin.zip"
checksum_sha256: "" # fill in after zipping — see below
tags:
- monitoring
```
```python
# plugins/myplugin/mcp_tools.py
from steward.mcp._context import current_caller, mcp_app
### Creating a release zip
The zip must contain the plugin files at the top level OR inside a single directory named after the plugin. Both layouts work:
async def list_targets(limit: int = 50) -> dict:
"""What this tool is for, when to use it over its siblings, what each
argument's empty value means, and what comes back. This docstring is the
tool's description: it is all the agent knows about it."""
app = mcp_app() # Quart's current_app does not exist here
async with app.db_sessionmaker() as db:
...
return {"targets": [...], "total": n, "truncated": n > limit}
```
# Layout A — flat (preferred)
myplugin.zip
├── plugin.yaml
├── __init__.py
└── ...
# Layout B — single top-level directory (also accepted, GitHub archive default)
myplugin.zip
└── myplugin/
├── plugin.yaml
└── ...
```
- `role` and `read_only` are required. A tool is listed and callable only for callers whose role reaches `role`, and only while the plugin is enabled.
- Tools run outside Quart's request pipeline: use `mcp_app()` and `current_caller()` from `steward.mcp._context`, never `current_app` or `session`.
- Refuse by raising `ValueError("what was wrong, and what to do instead")`. The agent reads that text. Any other exception reaches it only as a generic failure.
- Tool names are global across core and every plugin; a name already taken fails registration and the plugin contributes no tools (logged).
- Hot-reloading the plugin replaces its tools.
- Follow the response conventions the core tools use (`steward/mcp/tools/_shape.py`): times via `iso()` plus `age_s()`, lists through `bounded()` so a cut is never silent, series through `summarize()`, and lookups by name or id through `pick()`.
Generate the zip and its checksum:
### Contributing to get_overview
`get_overview` is the agent's first call: what is wrong right now, ranked. A plugin adds its own problems to it by exporting an optional `mcp_overview(db)` that returns findings built with `steward.mcp.tools.overview.finding`:
```python
# plugins/myplugin/__init__.py
async def mcp_overview(db) -> list[dict]:
from steward.mcp.tools.overview import finding
return [
finding("warning", "target_unreachable", t.name, f"{t.name} has not answered for {t.misses} scans",
since=t.last_ok_at, ids={"target_id": t.id}, next=f'get_target(target="{t.id}")')
for t in await _unreachable_targets(db)
]
```bash
cd steward-plugins
zip -r myplugin.zip myplugin/
sha256sum myplugin.zip # paste this into index.yaml checksum_sha256
```
- `severity` is `"critical"` (something is down or about to be) or `"warning"` (degraded, or heading that way). Report problems only; a healthy plugin returns `[]`.
- It is called only while the plugin is enabled, with its own session. If it raises, the overview still returns, and names the plugin under `unavailable`.
Upload `myplugin.zip` as a GitHub release asset, then update `index.yaml` with the release download URL and checksum.
---
### How install works
## Shipping a Plugin
When a user clicks **Install** in Settings → Plugins:
Steward never downloads plugins. There is no catalog and no install button: the app runs with no outbound network.
1. The app downloads the zip from `download_url`
2. Verifies the SHA-256 checksum (if provided)
3. Extracts the plugin into its `PLUGIN_DIR`
4. Runs any pending Alembic migrations for the plugin
5. Attempts a **hot-reload** — registers the blueprint and scheduled tasks without restarting
6. If the plugin was previously loaded (blueprint already mounted), a **restart** is required to pick up the new code
- **First-party plugins** live in this repository under `plugins/` and ship inside the image, so they version with core.
- **A third-party plugin** is a folder you put in the external plugin root, `/data/plugins` by default (`STEWARD_PLUGIN_DIR` moves it). Restart Steward and the plugin appears in Settings → Plugins, where you enable it. A bundled plugin of the same name wins over an external one.
Hot-reload works reliably for brand-new plugin installs. Updates to already-active plugins require a restart, which can be triggered from the same settings page.
---
@@ -412,6 +413,6 @@ Steward never downloads plugins. There is no catalog and no install button: the
- [ ] Revision IDs prefixed with plugin name
- [ ] `record_metric()` called inside `session.begin()`
- [ ] Routes use `@require_role` decorator
- [ ] MCP tools (if any) registered via `register_mcp_tools(register)` with a role and `read_only`
- [ ] Templates namespaced under `templates/myplugin/`
- [ ] Plugin enabled in app settings
- [ ] `index.yaml` entry added with `download_url` and `checksum_sha256`
docs/reference/code-map.md
-23
View File
@@ -89,7 +89,6 @@ Quick reference for where key functions, models, and entry points live in the co
| `AlertState` | `steward/models/alerts.py` | `alert_states` |
| `AlertEvent` | `steward/models/alerts.py` | `alert_events` |
| `PluginMetric` | `steward/models/metrics.py` | `plugin_metrics` |
| `PluginMetricLatest` | `steward/models/metrics.py` | `plugin_metric_latest` |
| `AnsibleRun` | `steward/models/ansible.py` | `ansible_runs` |
| `User` | `steward/models/users.py` | `users` |
| `AppSetting` | `steward/models/settings.py` | `app_settings` |
@@ -137,25 +136,3 @@ Quick reference for where key functions, models, and entry points live in the co
| `steward/migrations/versions/` | Core schema (hosts, users, monitors, alerts, app_settings) |
| `plugins/traefik/migrations/versions/` | `traefik_metrics` table |
| `alembic.ini` | Alembic config; `version_locations` lists all migration directories |
---
## MCP server and API tokens
| What | File | Function / Class |
|---|---|---|
| Mount `/mcp` (ASGI dispatch, bearer auth, DB gate) | `steward/mcp/server.py` | `mount_mcp(app)` |
| Server, role registry, refusals | `steward/mcp/server.py` | `StewardMCPServer`, `register_tool()`, `refusal()` |
| Plugin tools and overview findings | `steward/mcp/server.py` | `register_plugin(name, module)` |
| Caller and app inside a tool | `steward/mcp/_context.py` | `current_caller()`, `mcp_app()` |
| Core tool modules | `steward/mcp/tools/` | `register_all()`; one module per area |
| Response conventions | `steward/mcp/tools/_shape.py` | `iso`, `age_s`, `bounded`, `lookback`, `summarize`, `pick` |
| Audit a write as "<user> via MCP" | `steward/mcp/tools/_audit.py` | `audit_mcp()` |
| "What is wrong now" | `steward/mcp/tools/overview.py` | `get_overview()` |
| Playbooks, runs, schedules, alert actions | `steward/mcp/tools/automation.py` | `run_playbook()`, `wait_for_run()`, … |
| Monitor validation (forms and tools) | `steward/monitors/service.py` | `monitor_fields()`, `build_config()` |
| Alert rule / window validation | `steward/alerts/service.py` | `rule_fields()`, `window_fields()`, `acknowledge()` |
| Docker and host-agent tools | `plugins/docker/mcp_tools.py`, `plugins/host_agent/mcp_tools.py` | `register_tools()`, `mcp_overview()` |
| Token model | `steward/models/api_tokens.py` | `ApiToken` |
| Mint, resolve, revoke tokens | `steward/core/api_tokens.py` | `create_token()`, `resolve_token()`, `revoke_token()` |
| Token management page and connect card | `steward/ai_access/routes.py`, `steward/templates/ai_access/` | `ai_access_bp` |
plugins/docker/__init__.py
-19
View File
@@ -49,25 +49,6 @@ def get_blueprint():
return docker_bp
def register_mcp_tools(register) -> None:
from .mcp_tools import register_tools
register_tools(register)
async def mcp_overview(db) -> list[dict]:
from .mcp_tools import mcp_overview as findings
return await findings(db)
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (fleet-wide container view).
return [{"label": "Docker", "href": "/plugins/docker/"}]
def get_alert_metrics() -> dict[str, list[str]]:
"""What ingest.py records per container (resource "<host>/<container>"):
restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 /
0.0 from the container's HEALTHCHECK, only for containers that define one.
Per host (resource "<host>"), what Docker keeps on its disk (#5304)."""
from .disk_view import DISK_METRICS
return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS]}
plugins/docker/disk_view.py
-82
View File
@@ -1,82 +0,0 @@
# plugins/docker/disk_view.py
"""What Docker keeps on each host's disk, over time and by image repository
(#5304–#5306). Model-free, so it is unit-tested without a database.
The agent reports `docker system df` as current state; ingest.py records its
totals as metrics too (DISK_METRICS, resource = the host's name), so each
host's growth has a history and an alert rule can watch it.
"""
from __future__ import annotations
from datetime import datetime
# metric name → the df summary field it records; "disk_total_bytes" is their sum.
DISK_PARTS = {
"disk_images_bytes": "images_size",
"disk_containers_bytes": "containers_size",
"disk_volumes_bytes": "volumes_size",
"disk_build_cache_bytes": "build_cache_size",
}
DISK_METRICS = [*DISK_PARTS, "disk_total_bytes", "disk_reclaimable_bytes"]
def disk_metrics(disk: dict) -> dict[str, float]:
"""The metric samples one df report gives: each part, their total, and
what unused images hold."""
out = {m: float(int(disk.get(f) or 0)) for m, f in DISK_PARTS.items()}
out["disk_total_bytes"] = sum(out.values())
out["disk_reclaimable_bytes"] = float(int(disk.get("images_reclaimable") or 0))
return out
def growth(points: list[tuple[datetime, float]]) -> dict | None:
"""How much a total changed over the points given (oldest first): the
change, the first point's time and value. None with fewer than two."""
if len(points) < 2:
return None
(first_at, first), (_, last) = points[0], points[-1]
return {"change": last - first, "since": first_at, "from": first, "to": last,
"pct": (last - first) / first * 100 if first else None}
def image_repo(repo_tag: str | None) -> tuple[str | None, str | None]:
"""("ghcr.io/x/app", "1.2") for "ghcr.io/x/app:1.2"; a registry's port is
part of the repository ("reg:5000/app" has no tag); a digest reference
keeps its digest as the tag; (None, None) for an untagged image."""
if not repo_tag or repo_tag == "<none>" or repo_tag.startswith("<none>"):
return None, None
if "@" in repo_tag:
repo, _, digest = repo_tag.partition("@")
return repo, digest[:19]
head, sep, tag = repo_tag.rpartition(":")
if not sep or "/" in tag:
return repo_tag, None
return head, tag
def group_images(images: list[dict]) -> list[dict]:
"""Images folded under their repository, the most space first.
Each image: {repo_tag, image_id, size, shared_size, containers}. A group's
`own` counts each version's size less the layers it shares with other
images, so a repository is not charged twice for one base layer;
`reclaimable` is that, for the versions no container uses. Untagged images
form one group of their own (repo None)."""
groups: dict[str | None, dict] = {}
for im in images:
repo, tag = image_repo(im.get("repo_tag"))
g = groups.setdefault(repo, {"repo": repo, "versions": [], "own": 0, "size": 0,
"reclaimable": 0, "containers": 0})
own = max(int(im.get("size") or 0) - int(im.get("shared_size") or 0), 0)
used = int(im.get("containers") or 0)
g["versions"].append({"tag": tag, "image_id": im.get("image_id"), "size": int(im.get("size") or 0),
"shared": int(im.get("shared_size") or 0), "own": own, "containers": used})
g["own"] += own
g["size"] = max(g["size"], int(im.get("size") or 0))
g["containers"] += used
if not used:
g["reclaimable"] += own
for g in groups.values():
g["versions"].sort(key=lambda v: (v["containers"] == 0, -v["size"]))
g["unused"] = sum(1 for v in g["versions"] if not v["containers"])
return sorted(groups.values(), key=lambda g: (g["repo"] is None, -g["own"], g["repo"] or ""))
plugins/docker/ingest.py
+17 -155
View File
@@ -34,16 +34,12 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
"""Diff a fresh snapshot against stored per-container state → lifecycle events.
Pure function (no DB) so it's unit-testable. `old_state` maps name → the
previously stored {status, health, oom_killed, exit_code, restart_count}; `new_containers`
previously stored {status, health, oom_killed, exit_code}; `new_containers`
is the newest snapshot's list of container dicts. Returns a list of
(container_name, event, detail) tuples:
start — a container transitions into running (or a genuinely new
container appears already running)
die — also for a new container first seen already exited
non-zero: a task that crashed between two samples (#5049)
restart — Docker restarted it (its restart count went up); said
instead of a start, since that is what happened
stop — running → not-running, or a running container is removed
die — same as stop but with a non-zero exit code (abnormal)
oom — OOMKilled flips False→True
@@ -66,22 +62,14 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
old = old_state.get(name)
if old is None:
# Newly observed container: a start, or, when it is already down
# with a non-zero exit, the death that happened between samples.
# Newly observed container — only a start is meaningful (we have no
# prior state to diff a death against).
if new_running:
events.append((name, "start", c.get("image") or None))
elif new_status in _STOPPED_STATES and new_exit not in (None, 0):
events.append((name, "die", f"exit {new_exit}"))
continue
old_running = (old.get("status") or "").lower() == "running"
new_restarts = c.get("restart_count") or 0
old_restarts = old.get("restart_count") or 0
if new_restarts > old_restarts:
gained = new_restarts - old_restarts
events.append((name, "restart", f"restart #{new_restarts}" if gained == 1
else f"{gained} restarts, now #{new_restarts}"))
if new_running and not old_running and new_restarts <= old_restarts:
if new_running and not old_running:
events.append((name, "start", c.get("image") or None))
elif old_running and not new_running:
if new_exit not in (None, 0):
@@ -104,55 +92,6 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
return events
def _log_rows(log_batches, host_id: str):
"""Pure: flatten the agent's log batches → docker_logs row kwargs (no DB).
`log_batches` is a list of (recorded_at, records) where each record is
{"container", "stream", "ts", "line"}. A line's own Docker timestamp wins;
recorded_at is the fallback when the agent couldn't parse one. Drops the
agent's advisory truncation marker (container "_steward" — it signals a
deferral, not a real container), malformed records, and lineless records;
normalises an unknown stream to stdout. Unit-testable in isolation.
"""
for recorded_at, records in log_batches:
if not isinstance(records, list):
continue
for rec in records:
if not isinstance(rec, dict):
continue
name = rec.get("container")
line = rec.get("line")
if not name or name == "_steward" or line is None:
continue
ts = _parse_started_at(rec.get("ts")) or recorded_at
stream = rec.get("stream")
if stream not in ("stdout", "stderr"):
stream = "stdout"
yield {"host_id": host_id, "container_name": str(name)[:255],
"ts": ts, "stream": stream, "line": str(line)}
async def _persist_logs(session, host, log_batches) -> None:
"""Append pushed container log lines (one row per line) for this host.
Time-series / append-only — the per-container size+age ring (retention)
bounds growth, so a chatty container just keeps a shorter window. The global
toggle + exclude list (Settings, no restart) are enforced here: this is the
authoritative drop point, since the push model has no channel to tell an
agent to stop collecting.
"""
from steward.core.settings import get_setting
from .models import DockerLog
if not await get_setting(session, "docker.logs.enabled"):
return
exclude = set(await get_setting(session, "docker.logs.exclude") or ())
for row in _log_rows(log_batches, host.id):
if row["container_name"] in exclude:
continue
session.add(DockerLog(**row))
async def _persist_swarm(session, host, swarm: dict) -> None:
"""Upsert this manager's swarm topology; drop rows no longer reported.
@@ -181,7 +120,6 @@ async def _persist_swarm(session, host, swarm: dict) -> None:
row.running = int(s.get("running") or 0)
row.image = s.get("image")
row.placement_json = json.dumps(s.get("placement") or [])
row.tasks_json = json.dumps(s.get("tasks") or [])
row.updated_at = now
stale_services = delete(DockerSwarmService).where(
DockerSwarmService.host_id == host.id)
@@ -240,14 +178,6 @@ async def _persist_disk(session, host, disk: dict) -> None:
summary.containers_count = int(disk.get("containers_count") or 0)
summary.volumes_count = int(disk.get("volumes_count") or 0)
summary.updated_at = now
# The totals as metrics too (#5304): a history of each host's growth, and
# something an alert rule can watch. Resource: the host's name.
from steward.core.alerts import MetricBatch
from .disk_view import disk_metrics
batch = MetricBatch()
for metric, value in disk_metrics(disk).items():
batch.add(source_module="docker", resource_name=host.name, metric_name=metric, value=value)
await batch.record(session)
images = disk.get("images") or []
seen: set[str] = set()
@@ -271,37 +201,7 @@ async def _persist_disk(session, host, disk: dict) -> None:
await session.execute(stale)
async def _persist_status(session, host, status: dict) -> None:
"""Record whether Docker answered on this host, and why not.
A host reporting "unavailable" with no row yet never had Docker: nothing to
record. Once a host has had Docker, losing it is recorded, so its last
containers read as stale rather than as running."""
from .models import DockerHostStatus
row = await session.get(DockerHostStatus, host.id)
if row is None:
if status["status"] == "unavailable":
return
row = DockerHostStatus(host_id=host.id)
session.add(row)
if row.status != status["status"]:
row.since = status["at"]
row.status = status["status"]
row.detail = (status.get("detail") or "")[:255] or None
row.reported_at = status["at"]
async def _expire_swarm(session, host) -> None:
"""Drop the swarm rows a host reported while it was a manager."""
from .models import DockerSwarmNode, DockerSwarmService
await session.execute(delete(DockerSwarmService).where(DockerSwarmService.host_id == host.id))
await session.execute(delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id))
async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
logs=None, status=None, phases=None) -> None:
async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -> None:
"""Upsert containers + time-series + lifecycle events + swarm for one host.
`snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
@@ -311,39 +211,15 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
alert pipeline, and lifecycle-event derivation. `swarm` is the newest
sample's swarm object (or None off managers) — persisted when present.
`disk` is the newest sample's /system/df summary (or None on Docker-less
hosts) — persisted when present. `logs` is a list of (recorded_at, records)
log batches (m79) — appended one row per line when present, independent of
whether this sample also carried container metrics. `status` is the newest
sample's {"status", "detail", "at", "swarm_manager"} (agent ≥ 1.9.0): an
"ok" with an empty snapshot clears the host's containers, and a node that
says it is no longer a manager loses the swarm rows it reported as one.
`phases`, when given, is marked after each part (docker.status, .swarm,
.disk, .logs, .metrics, .events, .containers, .alerts, .reap), each
flushed before its mark, so a slow ingest names the part (#5181).
hosts) — persisted when present.
"""
from steward.core.alerts import MetricBatch
from steward.core.alerts import record_metric
from .models import DockerContainer, DockerEvent, DockerMetric
async def mark(name: str) -> None:
if phases is not None:
await session.flush()
phases.mark(f"docker.{name}")
if status is not None:
await _persist_status(session, host, status)
if status.get("swarm_manager") is False:
await _expire_swarm(session, host)
await mark("status")
if swarm is not None:
await _persist_swarm(session, host, swarm)
await mark("swarm")
if disk is not None:
await _persist_disk(session, host, disk)
await mark("disk")
if logs:
await _persist_logs(session, host, logs)
await mark("logs")
if not snapshots:
return
@@ -362,7 +238,6 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
mem_pct=c.get("mem_pct") or 0.0,
mem_usage_bytes=c.get("mem_usage_bytes") or 0,
))
await mark("metrics")
# Snapshot of stored per-container state BEFORE the upsert overwrites it —
# used to diff lifecycle events. Skip event derivation on the host's first
@@ -372,25 +247,17 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
)).scalars().all()
old_state = {
r.name: {"status": r.status, "health": r.health,
"oom_killed": r.oom_killed, "exit_code": r.exit_code,
"restart_count": r.restart_count}
"oom_killed": r.oom_killed, "exit_code": r.exit_code}
for r in old_rows
}
if old_state:
# A stop is dated when Docker says it happened (agent ≥ 1.11), not when
# the sample that noticed it was taken.
finished = {c.get("name"): _parse_started_at(c.get("finished_at"))
for c in latest_containers}
for name, event, detail in _derive_events(old_state, latest_containers):
at = finished.get(name) if event in ("die", "stop") else None
session.add(DockerEvent(
host_id=host.id, container_name=name,
event=event, detail=detail, at=at or latest_at,
event=event, detail=detail, at=latest_at,
))
await mark("events")
# Current state + alerts from the newest snapshot only.
batch = MetricBatch()
for c in latest_containers:
name = c.get("name")
if not name:
@@ -428,19 +295,19 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
# on different hosts don't collide in the metric/alert namespace.
resource = f"{host.name}/{name}"
if c.get("status") == "running" and c.get("cpu_pct") is not None:
batch.add(
source_module="docker",
await record_metric(
session=session, source_module="docker",
resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
)
if c.get("mem_pct") is not None:
batch.add(
source_module="docker",
await record_metric(
session=session, source_module="docker",
resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
)
# Restart count is alertable regardless of state (crash-looping matters
# most while the container is down/restarting).
batch.add(
source_module="docker",
await record_metric(
session=session, source_module="docker",
resource_name=resource, metric_name="restart_count",
value=float(c.get("restart_count", 0) or 0),
)
@@ -449,16 +316,12 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
# container).
health = c.get("health")
if health in ("healthy", "unhealthy", "starting"):
batch.add(
source_module="docker",
await record_metric(
session=session, source_module="docker",
resource_name=resource, metric_name="is_healthy",
value=1.0 if health == "healthy" else 0.0,
)
await mark("containers")
await batch.record(session)
await mark("alerts")
# Reap containers that vanished from the host's latest listing. Without this,
# removed one-shot containers (CI job runners, buildkit builders, codex jobs)
# accumulate forever as permanent "stopped" rows and inflate every count.
@@ -471,4 +334,3 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
if seen_names:
reap = reap.where(DockerContainer.name.notin_(seen_names))
await session.execute(reap)
await mark("reap")
plugins/docker/mcp_tools.py
-642
View File
@@ -1,642 +0,0 @@
"""The docker plugin's MCP tools and its get_overview findings.
Registered through register_mcp_tools() in __init__.py, so they exist only
while the plugin is enabled. Reads go through service.py, the same functions
the plugin's pages use.
"""
from __future__ import annotations
import json
from dataclasses import dataclass
from datetime import datetime, timedelta
from sqlalchemy import select
from steward.ansible.runner import trigger_run
from steward.ansible.sources import BUILTIN_SOURCE_NAME
from steward.mcp._context import current_caller, mcp_app
from steward.mcp.tools._audit import audit_mcp
from steward.mcp.tools._shape import (
age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, summarize,
)
from steward.mcp.tools.overview import finding
from steward.models.ansible_inventory import AnsibleTarget
from steward.models.hosts import Host
from steward.models.users import UserRole
from . import service as docker_svc
from .service import PROBLEM_SAYS, PROBLEM_SEVERITY, PROBLEMS, container_problems, problem_since
from .dedup import dedup_by_container_id
from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
from .swarm_view import build_swarm_services, replaced_tasks, task_time
CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
LOG_LINE_CHARS = 2000
# ── pure helpers ─────────────────────────────────────────────────────────────
def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
"""Keep both ends of an over-long line and say how much of the middle went."""
if text is None or len(text) <= limit:
return text
keep = limit // 2
return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
def container_row(c, host_names: dict, problems: list[str], now: datetime,
replaced: bool = False, since: datetime | None = None) -> dict:
running = (c.status or "").lower() == "running"
return {
"container_name": c.name,
"host_id": c.host_id,
"host_name": host_names.get(c.host_id),
"image": c.image,
"status": c.status,
"health": c.health,
"problems": problems,
# When the worst problem began, from the container's retained events.
"problem_since": iso(since),
"problem_age_s": age_s(since, now),
"exit_code": c.exit_code,
"oom_killed": bool(c.oom_killed),
"restart_count": c.restart_count,
"cpu_pct": rnd(c.cpu_pct),
"mem_pct": rnd(c.mem_pct),
"mem_usage_bytes": c.mem_usage_bytes,
"mem_limit_bytes": c.mem_limit_bytes,
"started_at": iso(c.started_at),
"uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
"compose_project": c.compose_project,
"swarm_service": c.service_name,
# A swarm task Swarm has replaced: history, judged on its service.
"swarm_replaced": replaced if c.service_name else None,
"reported_at": iso(c.scraped_at),
"reported_age_s": age_s(c.scraped_at, now),
}
def _from_iso(text: str | None) -> datetime | None:
return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
def container_findings(rows: list[dict]) -> list[dict]:
out = []
for r in rows:
if not r["problems"]:
continue
worst = r["problems"][0]
where = f" on {r['host_name']}" if r["host_name"] else ""
detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
out.append(finding(
PROBLEM_SEVERITY[worst], f"container_{worst}", r["container_name"],
f"container {r['container_name']}{where} {' and '.join(PROBLEM_SAYS[p] for p in r['problems'])}{detail}",
since=_from_iso(r["problem_since"]),
ids={"host_id": r["host_id"], "container_name": r["container_name"]},
next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
return out
def task_error_text(err: dict | None) -> str:
"""Swarm's own account of a task's end, as one clause."""
if not err:
return ""
why = err.get("error") or err.get("message") or err.get("state") or "unknown"
code = f", exit {err['exit_code']}" if err.get("exit_code") not in (None, 0) else ""
where = f" on {err['node']}" if err.get("node") else ""
return f"{why}{code}{where}"
def service_findings(services: list[dict]) -> list[dict]:
"""Swarm services that are failing: crash-looping, under-replicated, or that
lost a task to an error in the last 24h. Each names Swarm's last task error."""
out = []
for s in services:
replicas = f"{s['running']} of {s['desired']} replicas running"
last = task_error_text(s.get("last_error"))
tail = f"; last task error: {last}" if last else ""
ids = {"service_name": s["name"]}
since = s.get("trouble_since")
if s.get("crash_looping"):
out.append(finding(
"critical", "swarm_service_crash_looping", s["name"],
f"swarm service {s['name']} keeps failing: {s['failures_1h']} tasks failed in the "
f"last hour, {replicas}{tail}", since=since, ids=ids,
next="list_swarm_services()"))
elif s["state"] != "healthy":
out.append(finding(
"critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
s["name"], f"swarm service {s['name']} has {replicas}{tail}",
since=since, ids=ids, next="list_swarm_services()"))
elif s.get("failures_24h"):
n = s["failures_24h"]
out.append(finding(
"warning", "swarm_task_failed", s["name"],
f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
f"24h and is now {replicas}{tail}", since=since, ids=ids,
next="list_swarm_services()"))
return out
# ── loading ──────────────────────────────────────────────────────────────────
async def _host_names(db) -> dict[str, str]:
return dict((await db.execute(select(Host.id, Host.name))).all())
def _crash_loop_deaths() -> int:
return docker_svc.crash_loop_deaths(mcp_app().config)
async def _replaced(db, containers) -> set[tuple]:
services = list((await db.execute(select(DockerSwarmService))).scalars())
return replaced_tasks(containers, services)
async def _rows(db, now: datetime) -> list[dict]:
"""Every container with its problems. A swarm task Swarm replaced has none:
its failure, if it was one, belongs to its service (service_findings)."""
host_names = await _host_names(db)
return [container_row(r["container"], host_names, r["problems"], now,
replaced=r["replaced"], since=r["since"])
for r in await docker_svc.ranked_containers(db, now, _crash_loop_deaths())]
async def _swarm(db, now: datetime):
containers = dedup_by_container_id(await docker_svc.list_containers(db))
services = list((await db.execute(select(DockerSwarmService))).scalars())
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
host_names = await _host_names(db)
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
built = build_swarm_services(containers, services, node_hostname, host_names,
now=now, crash_loop_deaths=_crash_loop_deaths())
return built["services"], nodes, host_names
async def mcp_overview(db) -> list[dict]:
"""get_overview findings: failing containers and under-replicated swarm services."""
now = now_utc()
rows = await _rows(db, now)
services, _, _ = await _swarm(db, now)
return container_findings(rows) + service_findings(services)
# ── tools ────────────────────────────────────────────────────────────────────
async def list_containers(host: str = "", status: str = "", problem: str = "",
compose_project: str = "", limit: int = 100) -> dict:
"""Docker containers across every host that runs a Steward agent with Docker.
host: a host's name or id (list_hosts); empty for all hosts.
status: Docker's state — "running", "exited", "restarting", "paused",
"created" or "dead"; empty for all.
problem: "any" for containers with something wrong, or one of
"restarting", "crash_looping" (died repeatedly within the last hour;
the count is set under Settings → Thresholds, default 3), "unhealthy"
(failing its healthcheck), "oom_killed", "exited_nonzero" (died with an
error in the last 24h; exit 137/143 without an OOM kill is a stop, not
an error); empty for no filter.
compose_project: only containers from this Compose project.
limit: default 100, max 500.
Containers with problems come first. Each row's `problems` lists what is
wrong, most severe first, and `problem_since` / `problem_age_s` say when
the worst of them began (null when Steward's retained events cannot say).
`reported_age_s` is how old the agent's last report is, so a large value
means the row may be stale.
A swarm task's container that Swarm has replaced (a rolling update, or a
retry after a failure) has `swarm_replaced: true` and no problems: Swarm
restarts a task as a new container, so failures are counted on the
service — list_swarm_services shows them.
"""
if problem and problem != "any" and problem not in PROBLEMS:
raise ValueError(f"problem {problem!r} is not 'any' or one of {', '.join(PROBLEMS)}.")
limit = clamp_limit(limit, 100, 500)
now = now_utc()
async with db_session() as db:
rows = await _rows(db, now)
if host:
hosts = list((await db.execute(select(Host))).scalars())
h = pick(host, hosts, kind="host", list_tool="list_hosts")
rows = [r for r in rows if r["host_id"] == h.id]
if status:
rows = [r for r in rows if (r["status"] or "").lower() == status.lower()]
if problem == "any":
rows = [r for r in rows if r["problems"]]
elif problem:
rows = [r for r in rows if problem in r["problems"]]
if compose_project:
rows = [r for r in rows if r["compose_project"] == compose_project]
rows.sort(key=lambda r: (not r["problems"], (r["status"] or "") != "running",
(r["host_name"] or "").lower(), r["container_name"]))
return bounded(rows, limit, key="containers",
more="Pass a larger `limit` (max 500), or filter by `host`, `status` or `problem`.")
@dataclass
class RemovedContainer:
"""A container Docker has removed, known only by its retained logs and events."""
host_id: str
name: str
last_seen: datetime
_REMOVED_HINT = ("It may have been removed: Swarm replaces a dead task with a new container under "
"a new name. list_containers(status=\"exited\") lists stopped containers, and "
"search_container_logs(host=...) searches a whole host.")
async def _pick_container(db, container: str, host: str, *, removed: bool = False):
"""The container named `container`: a live one, else (with `removed`) one
Docker has removed whose logs or events Steward still holds, returned as a
RemovedContainer. Exact names beat partial ones, and live beats removed."""
containers = await docker_svc.list_containers(db)
host_id = None
if host:
hosts = list((await db.execute(select(Host))).scalars())
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
containers = [c for c in containers if c.host_id == host_id]
want = (container or "").strip()
if not want:
raise ValueError("pass a container name; list_containers lists them.")
live = {(c.host_id, c.name) for c in containers}
def _one(found, what):
if len(found) > 1:
raise ValueError(f"{len(found)} hosts have {what} named {want!r}; pass `host` "
f"(one of: {', '.join(sorted(f[0] for f in found))}).")
return found[0] if found else None
same = [c for c in containers if c.name == want] or \
[c for c in containers if c.name.lower() == want.lower()]
if same:
return _one([(c.host_id, c) for c in same], "a container")[1]
if removed:
gone = [r for r in await docker_svc.retained_names(db, want, exact=True, host_id=host_id)
if (r[0], r[1]) not in live]
if gone:
return RemovedContainer(*_one(gone, "a removed container"))
# One more chance: a unique container whose name contains it (swarm task
# names look like "stack_svc.1.k2j…").
partial = [c for c in containers if want.lower() in c.name.lower()]
if len(partial) == 1:
return partial[0]
gone = []
if removed and not partial:
gone = [r for r in await docker_svc.retained_names(db, want, exact=False, host_id=host_id)
if (r[0], r[1]) not in live]
if len(gone) == 1:
return RemovedContainer(*gone[0])
names = sorted(c.name for c in partial)[:10] + [f"{r[1]} (removed)" for r in gone[:10]]
hint = f" Containers whose names contain it: {', '.join(names)}." if names else ""
raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} "
f"{'' if names else _REMOVED_HINT + ' '}list_containers lists them.")
def _removed_note(c: RemovedContainer) -> str:
return (f"{c.name} no longer exists: Docker removed it (Swarm replaces a failed task with a new "
f"container). This is what Steward retained; it was last seen {iso(c.last_seen)}.")
async def get_container(container: str, host: str = "", range: str = "24h", events: int = 20) -> dict:
"""One container in depth: state, resources over time and lifecycle events.
container: the container's name, as list_containers shows. A unique part of
a name also works (swarm task names are long).
host: the host's name or id. Needed only when several hosts run a
container with this name.
range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
"7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
events: how many lifecycle events (start, restart, stop, die, oom, health_change)
to return, newest first (default 20, max 100).
A container Docker has since removed (a dead Swarm task, a `docker rm`)
still resolves by its exact name, or by a part of it that matches one
removed container, while Steward retains its logs or events: the reply
has `removed: true`, `last_seen_at`, its history and its events, and no
live state. A live container has `removed: false`.
"""
since, key = lookback(range)
events = clamp_limit(events, 20, 100)
now = now_utc()
async with db_session() as db:
c = await _pick_container(db, container, host, removed=True)
if isinstance(c, RemovedContainer):
return await _removed_container(db, c, since, key, events, now)
gone = (c.host_id, c.name) in await _replaced(db, await docker_svc.list_containers(db))
failures = (await docker_svc.recent_failures_by_container(
db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
host_names = await _host_names(db)
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)])
hour_ago = now - timedelta(hours=1)
recent = sum(1 for e in failures if e.at >= hour_ago)
try:
ports = json.loads(c.ports_json or "[]")
except ValueError:
ports = []
problems = [] if gone else container_problems(c, recent, bool(failures), _crash_loop_deaths())
began = problem_since(problems, failures, hour_ago, sick.get((c.host_id, c.name)))
return {
**container_row(c, host_names, problems, now, replaced=gone, since=began),
"removed": False,
"container_id": c.container_id,
"ports": ports,
"net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
"blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
"range": key,
"cpu_pct_history": summarize(cpu),
"mem_pct_history": summarize(mem),
"events": _event_rows(evs, now),
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
}
def _event_rows(evs, now: datetime) -> list[dict]:
return [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)}
for e in evs]
async def _removed_container(db, c: RemovedContainer, since, key: str, events: int,
now: datetime) -> dict:
"""get_container for a container Docker has removed: what Steward kept."""
host_names = await _host_names(db)
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
return {
"container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id),
"removed": True, "note": _removed_note(c),
"last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now),
"range": key,
"cpu_pct_history": summarize(cpu),
"mem_pct_history": summarize(mem),
"events": _event_rows(evs, now),
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")',
}
async def list_swarm_services(limit: int = 100) -> dict:
"""Docker Swarm services with their replicas per node, and the swarm's nodes.
`state` is "healthy" (all desired replicas running), "degraded" (some) or
"down" (none). `replicas` are the service's current tasks: each names its
host and container status, and one marked `ghost` runs on a node with no
Steward agent, so Steward knows only how many tasks are placed there.
Tasks Swarm replaced (rolling updates, retries) are left out and counted
in `previous_tasks`.
From Swarm's own task records (a manager's agent 1.8.0 or later;
`tasks_known: false` and nulls otherwise):
- `failures_1h` / `failures_24h`: tasks that failed or were rejected.
Swarm retries a failed task as a new one, so this, not any container's
restart count, is how a crash loop shows. Swarm keeps about 5 old tasks
per replica, which bounds the count.
- `crash_looping`: failures_1h reached the crash-loop count set under
Settings → Thresholds (default 3).
- `last_error`: the newest task that failed, was rejected, or carries an
error while Swarm retries it — `error` is Swarm's own text (e.g. "No
such image", "task: non-zero exit (1)"), with `exit_code`, `node`, `at`.
- `trouble_since` / `trouble_age_s`: when the current trouble began — a
crash loop's first failure in the hour, a short service's first failed
task in its failing slots, a recovered service's first failure in 24h.
Swarm keeps few old tasks, so a long outage reads as a lower bound.
Services with a problem come first. `nodes` lists every swarm node once,
with its role (manager/worker), availability and status.
limit: default 100, max 500.
"""
limit = clamp_limit(limit, 100, 500)
now = now_utc()
async with db_session() as db:
services, nodes, host_names = await _swarm(db, now)
seen, node_rows = set(), []
for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
if n.node_id in seen:
continue
seen.add(n.node_id)
node_rows.append({"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
"leader": n.leader, "availability": n.availability, "status": n.status,
"reported_by": host_names.get(n.host_id)})
node_rows.sort(key=lambda n: n["hostname"].lower())
order = {"down": 0, "degraded": 1, "healthy": 2}
def _rank(s):
return (not s["crash_looping"], order.get(s["state"], 3), not s["failures_24h"], s["name"])
def _error_row(err):
if err is None:
return None
at = task_time(err)
return {"error": err["error"], "message": err["message"], "state": err["state"],
"exit_code": err["exit_code"], "node": err["node"],
"at": iso(at), "age_s": age_s(at, now) if at else None}
rows = [
{
"service_name": s["name"], "mode": s["mode"], "state": s["state"],
"running": s["running"], "desired": s["desired"],
"crash_looping": s["crash_looping"],
"failures_1h": s["failures_1h"], "failures_24h": s["failures_24h"],
"last_error": _error_row(s["last_error"]),
"trouble_since": iso(s["trouble_since"]),
"trouble_age_s": age_s(s["trouble_since"], now),
"previous_tasks": s["previous_tasks"], "tasks_known": s["tasks_known"],
"replicas": [
{"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
{"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
"status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
"cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
for r in s["replicas"]
],
}
for s in sorted(services, key=_rank)
]
return {"nodes": node_rows, **bounded(rows, limit, key="services")}
async def search_container_logs(container: str = "", host: str = "", text: str = "",
range: str = "1h", stream: str = "", before: str = "",
context: int = 0, limit: int = 50) -> dict:
"""Search the container log lines Steward has retained, newest first.
container: one container's name (as list_containers shows); empty searches
every container (narrow with `host`). A container Docker has removed
still works while its lines are retained: the reply then has
`container_removed: true` and a `note` saying when it was last seen.
host: a host's name or id; empty for all hosts.
text: case-insensitive substring to match; empty matches every line.
range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are
kept per container up to a size and age limit, so older lines may be gone.
stream: "stdout" or "stderr"; empty for both.
before: an ISO time; returns only lines older than it. Pass a response's
`next_before` to page further back.
context: lines of surrounding log (0-5) to include with each match, from
the same container. Only with `container` set.
limit: matches to return, default 50, max 200.
Lines longer than 2000 characters keep both ends, and the middle is
replaced by a marker saying how much was cut.
"""
since, key = lookback(range)
limit = clamp_limit(limit, 50, 200)
context = max(0, min(int(context or 0), 5))
if context and not container:
raise ValueError("context needs `container`: it shows lines around a match within one container's log.")
if stream and stream not in ("stdout", "stderr"):
raise ValueError(f"stream {stream!r} is not 'stdout' or 'stderr'.")
before_dt = None
if before:
try:
before_dt = datetime.fromisoformat(before.replace("Z", "+00:00"))
except ValueError as exc:
raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc
async with db_session() as db:
host_id = name = removed = None
if container:
c = await _pick_container(db, container, host, removed=True)
host_id, name = c.host_id, c.name
removed = c if isinstance(c, RemovedContainer) else None
elif host:
hosts = list((await db.execute(select(Host))).scalars())
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
rows = await docker_svc.search_logs(db, since=since, host_id=host_id, name=name, text=text,
stream=stream, before=before_dt, limit=limit + 1)
more = len(rows) > limit
rows = rows[:limit]
host_names = await _host_names(db)
matches = []
for r in rows:
m = {"at": iso(r.ts), "container_name": r.container_name, "host_id": r.host_id,
"host_name": host_names.get(r.host_id), "stream": r.stream, "line": cut_middle(r.line)}
if context:
pre, post = await docker_svc.log_context(db, r.host_id, r.container_name, r.ts, context)
m["before"] = [cut_middle(x.line) for x in pre]
m["after"] = [cut_middle(x.line) for x in post]
matches.append(m)
out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more}
if removed is not None:
out["container_removed"] = True
out["note"] = _removed_note(removed)
if removed.last_seen < since:
out["note"] += " That is before this range; pass a longer `range`."
if more:
# Full precision: `at` is cut to the second for reading, and a cursor
# cut the same way would skip lines logged earlier in that second.
out["next_before"] = rows[-1].ts.isoformat()
out["more"] = "More matches are older; pass next_before as `before`, or narrow with `text`."
return out
async def get_docker_disk_usage(host: str = "") -> dict:
"""Docker disk usage per host: images, containers, volumes and build cache, in bytes.
host: a host's name or id; empty for every host. `images_reclaimable_bytes`
is what pruning unused images would free. An admin can prune from
Steward's Docker → Disk page.
"""
async with db_session() as db:
summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
host_names = await _host_names(db)
if host:
hosts = list((await db.execute(select(Host))).scalars())
hid = pick(host, hosts, kind="host", list_tool="list_hosts").id
summaries = [s for s in summaries if s.host_id == hid]
rows = [
{
"host_id": s.host_id, "host_name": host_names.get(s.host_id),
"images_total": s.images_total, "images_active": s.images_active,
"images_bytes": s.images_size, "images_reclaimable_bytes": s.images_reclaimable,
"containers_count": s.containers_count, "containers_bytes": s.containers_size,
"volumes_count": s.volumes_count, "volumes_bytes": s.volumes_size,
"build_cache_bytes": s.build_cache_size,
"reported_at": iso(s.updated_at), "reported_age_s": age_s(s.updated_at),
}
for s in summaries
]
rows.sort(key=lambda r: -(r["images_reclaimable_bytes"] or 0))
return {"hosts": rows, "total": len(rows)}
PRUNE_BUILTIN_PLAYBOOK = "maintenance/docker_prune.yml"
_STOPPED = ("exited", "created", "dead")
def prune_preview(what: str, usage, stopped: int) -> dict:
"""What a prune of `what` would remove, from the host's latest disk report."""
out: dict = {}
if what in ("containers", "system"):
out["stopped_containers"] = stopped
if what == "images":
out["unused_images_bytes"] = usage.images_reclaimable if usage else None
if what == "system":
out["dangling_images"] = "yes (unused but tagged images stay)"
out["build_cache_bytes"] = usage.build_cache_size if usage else None
out["unused_networks"] = "yes"
return out
async def prune_docker(host: str, what: str, check: bool = True) -> dict:
"""Reclaim Docker disk on a host. Previews unless check=false.
host: the Steward host's name or id. It needs a linked Ansible target
(list_inventory shows the links); the prune runs through Ansible as
Steward's built-in docker_prune playbook.
what: "containers" removes stopped containers; "images" removes ALL
unused images, not only dangling ones; "system" is `docker system
prune -f`: stopped containers, dangling images, unused networks and
build cache. Volumes are never pruned.
check: true (default) starts nothing and returns `would_remove`, sized
from the host's latest Docker disk report. Ansible's own check mode
skips the prune commands, so a check-mode run would show nothing.
Pass false to run the prune.
Admin only, as on the Docker → Disk page. A real prune returns `run_id`
(follow it with wait_for_run) and is recorded in the audit log.
"""
what = (what or "").strip()
if what not in docker_svc.PRUNE_TARGETS:
raise ValueError(f"what {what!r} is not one of {', '.join(docker_svc.PRUNE_TARGETS)}.")
async with db_session() as db:
h = pick(host, (await db.execute(select(Host))).scalars(), kind="host", list_tool="list_hosts")
target = (await db.execute(
select(AnsibleTarget).where(AnsibleTarget.host_id == h.id))).scalars().first()
usage = await db.get(DockerDiskUsage, h.id)
stopped = sum(1 for c in await docker_svc.list_containers(db, h.id)
if (c.status or "").lower() in _STOPPED)
if target is None:
raise ValueError(f"host {h.name!r} has no linked Ansible target, so Steward cannot reach it "
"to prune. Link one under Ansible → Inventory. Nothing was run.")
preview = {
"host": h.name,
"what": what,
"ansible_target": target.name,
"would_remove": prune_preview(what, usage, stopped),
"disk_report_age_s": age_s(usage.updated_at) if usage else None,
}
if check:
return {**preview, "check": True,
"summary": f"Nothing was run. Pass check=false to prune {what} on {h.name}."}
caller = current_caller()
run, _src, err = await trigger_run(
mcp_app(), source_name=BUILTIN_SOURCE_NAME, playbook_path=PRUNE_BUILTIN_PLAYBOOK,
inventory_scope=f"steward:target:{target.id}",
params={"extra_vars_map": docker_svc.prune_extra_vars(what)},
triggered_by=caller.user_id, origin="mcp")
if err:
raise ValueError(f"the prune could not start: {err}.")
await audit_mcp("docker.prune.started", "host", h.name,
{"host_id": h.id, "what": what, "run_id": run.id})
return {**preview, "check": False, "run_id": run.id,
"next": f'wait_for_run(run_id="{run.id}")'}
def register_tools(register) -> None:
for fn in (list_containers, get_container, list_swarm_services,
search_container_logs, get_docker_disk_usage):
register(fn, role=UserRole.viewer, read_only=True)
register(prune_docker, role=UserRole.admin, read_only=False)
plugins/docker/migrations/versions/docker_009_container_logs.py
-48
View File
@@ -1,48 +0,0 @@
"""Docker container logs table
Adds docker_logs — one row per container log line, pushed by the host agent and
folded into its metrics push. Time-series, host-scoped (container names are only
unique within a host). Bounded by a per-container size+age ring in retention, so
a chatty container keeps a shorter window rather than growing without limit.
Additive create_table + twin indexes (viewer lookup by (host, container, ts);
age-cutoff prune by ts).
Revision ID: docker_009_container_logs
Revises: docker_008_bigint_mem
Create Date: 2026-07-19
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_009_container_logs"
down_revision: Union[str, None] = "docker_008_bigint_mem"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"docker_logs",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("host_id", sa.String(length=36), nullable=False),
sa.Column("container_name", sa.String(length=255), nullable=False),
sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
sa.Column("stream", sa.String(length=8), nullable=False, server_default="stdout"),
sa.Column("line", sa.Text(), nullable=False, server_default=""),
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_docker_logs_host_id", "docker_logs", ["host_id"])
op.create_index("ix_docker_logs_container_name", "docker_logs", ["container_name"])
op.create_index("ix_docker_logs_host_container_time",
"docker_logs", ["host_id", "container_name", "ts"])
op.create_index("ix_docker_logs_ts", "docker_logs", ["ts"])
def downgrade() -> None:
op.drop_index("ix_docker_logs_ts", table_name="docker_logs")
op.drop_index("ix_docker_logs_host_container_time", table_name="docker_logs")
op.drop_index("ix_docker_logs_container_name", table_name="docker_logs")
op.drop_index("ix_docker_logs_host_id", table_name="docker_logs")
op.drop_table("docker_logs")
plugins/docker/migrations/versions/docker_010_swarm_tasks.py
-30
View File
@@ -1,30 +0,0 @@
"""Docker swarm service tasks column
Adds docker_swarm_services.tasks_json — each service's newest tasks as a
manager's agent (≥ 1.8.0) reports them: slot, node, desired and current state,
Swarm's error and message, exit code and time. Lets Steward tell a task Swarm
replaced from one that crashed, count a service's failures across task
retries, and say why its last task failed. Additive column; no DROP+recreate.
Revision ID: docker_010_swarm_tasks
Revises: docker_009_container_logs
Create Date: 2026-10-06
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_010_swarm_tasks"
down_revision: Union[str, None] = "docker_009_container_logs"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("docker_swarm_services",
sa.Column("tasks_json", sa.Text, nullable=False,
server_default="[]"))
def downgrade() -> None:
op.drop_column("docker_swarm_services", "tasks_json")
plugins/docker/migrations/versions/docker_011_host_status.py
-36
View File
@@ -1,36 +0,0 @@
"""Docker host status
Adds docker_host_status: per host, whether Docker answered at the agent's
last report (ok / unavailable / error) and why not. Lets Steward show a host
whose Docker went away as such, instead of its last containers as running
(#5152). Additive create_table.
Revision ID: docker_011_host_status
Revises: docker_010_swarm_tasks
Create Date: 2026-10-06
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_011_host_status"
down_revision: Union[str, None] = "docker_010_swarm_tasks"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"docker_host_status",
sa.Column("host_id", sa.String(length=36), nullable=False),
sa.Column("status", sa.String(length=16), nullable=False),
sa.Column("detail", sa.String(length=255), nullable=True),
sa.Column("since", sa.DateTime(timezone=True), nullable=False),
sa.Column("reported_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("host_id"),
)
def downgrade() -> None:
op.drop_table("docker_host_status")
plugins/docker/models.py
+5 -68
View File
@@ -1,5 +1,6 @@
# plugins/docker/models.py
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from sqlalchemy import (
BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text,
@@ -7,7 +8,6 @@ from sqlalchemy import (
)
from sqlalchemy.orm import Mapped, mapped_column
from steward.models.base import Base
from steward.core.ids import time_ordered_id
class DockerContainer(Base):
@@ -67,7 +67,7 @@ class DockerMetric(Base):
__tablename__ = "docker_metrics"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=time_ordered_id
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
@@ -103,7 +103,7 @@ class DockerMetricHourly(Base):
__tablename__ = "docker_metrics_hourly"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=time_ordered_id
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
@@ -139,14 +139,14 @@ class DockerEvent(Base):
__tablename__ = "docker_events"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=time_ordered_id
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
)
container_name: Mapped[str] = mapped_column(String(255), nullable=False)
event: Mapped[str] = mapped_column(String(16), nullable=False)
# start | restart | stop | die | oom | health_change
# start | stop | die | oom | health_change
detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
# e.g. "exit 137", "healthy→unhealthy", image on start
at: Mapped[datetime] = mapped_column(
@@ -163,43 +163,6 @@ class DockerEvent(Base):
)
class DockerLog(Base):
"""Container log lines pushed by the host agent — one row per line.
Time-series, scoped to the reporting host (container names are only unique
within a host, same identity as docker_metrics). The agent tails each running
container incrementally and folds new lines into its metrics push; `ts` is the
line's own Docker timestamp. Bounded by a per-container size+age ring
(retention), so a chatty container just keeps a shorter window rather than
growing without limit.
"""
__tablename__ = "docker_logs"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=time_ordered_id
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
)
container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
ts: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
)
stream: Mapped[str] = mapped_column(String(8), nullable=False, default="stdout")
# stdout | stderr
line: Mapped[str] = mapped_column(Text, nullable=False, default="")
# Viewer filters on (host_id, container_name) and sorts by time; the ring
# prune walks the same key newest-first. A second index on `ts` alone serves
# the age-cutoff delete. Twin-index idiom, mirroring docker_events.
__table_args__ = (
Index("ix_docker_logs_host_container_time",
"host_id", "container_name", "ts"),
Index("ix_docker_logs_ts", "ts"),
)
class DockerSwarmService(Base):
"""A Swarm service as seen by a manager host: desired vs running replicas.
@@ -223,11 +186,6 @@ class DockerSwarmService(Base):
# A manager sees every task, so this captures cross-node placement that the
# local docker_containers rows (this host only) can't.
placement_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
# The service's newest tasks, newest first, as the manager's agent shaped
# them: [{"task_id", "slot", "node_id", "desired_state", "state", "message",
# "error", "exit_code", "at"}], JSON. Swarm's own record of which tasks it
# replaced and why each one stopped; "[]" from agents older than 1.8.0.
tasks_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
@@ -304,24 +262,3 @@ class DockerSwarmNode(Base):
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
)
class DockerHostStatus(Base):
"""Whether Docker answered on a host at its agent's last report (agent ≥ 1.9.0).
"ok" when it did; "unavailable" when there is no Docker to ask (no socket,
nothing listening); "error" when Docker is there and the call failed.
`detail` says why for the last two. While a host is not "ok" its container
rows are what Docker last said, not what is true now, and read as stale.
A host that never had Docker gets no row.
"""
__tablename__ = "docker_host_status"
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
)
status: Mapped[str] = mapped_column(String(16), nullable=False)
detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
# When the status last changed, and the agent's newest report.
since: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
reported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
plugins/docker/overview.py
-150
View File
@@ -1,150 +0,0 @@
# plugins/docker/overview.py
"""The Docker page's two tables, as plain rows (#5299, #5301): one row per
swarm service and one per standalone container, with the search, stack, host
and status filters and the column sort the page offers. Model-free, so it is
unit-tested without a database; routes.rows() feeds it.
A stack is how the operator thinks of a group of services. Swarm names a
stack's services "<stack>_<service>", so a service's stack is the part of its
name before the first "_"; a standalone container's is its compose project.
Stacks filter the list rather than section it, as host groups do on Hosts.
"""
from __future__ import annotations
from datetime import datetime
from .swarm_view import service_trouble
# Status filter values and their labels; NO_STACK picks rows with no stack.
DOCKER_STATES = {"problem": "Needs attention", "healthy": "Healthy", "stopped": "Stopped"}
NO_STACK = "__none__"
# Each sort and whether it starts descending: the busiest first; "up" starts
# with the shortest uptime, the thing that most recently restarted.
DOCKER_SORTS = {"status": False, "name": False, "stack": False, "cpu": True, "mem": True,
"up": False}
def stack_of(service_name: str) -> str | None:
"""The stack a swarm service was deployed in: "arr" for "arr_radarr";
None for a name with no "_" (a service created on its own)."""
stack, sep, rest = service_name.partition("_")
return stack if sep and stack and rest else None
def _seconds_up(started_at, now: datetime) -> float | None:
return None if started_at is None else max((now - started_at).total_seconds(), 0.0)
def service_row(s: dict, now: datetime) -> dict:
"""A built swarm service (swarm_view.build_swarm_services) as a table row.
CPU and memory add up the running replicas; "up" is the newest replica's
start, so a service that just restarted a task reads as just restarted."""
reals = [r for r in s["replicas"] if not r["ghost"]]
running = [r for r in reals if (r["status"] or "").lower() == "running"]
cpu = [r["cpu_pct"] for r in running if r.get("cpu_pct") is not None]
mem = [r["mem_bytes"] for r in running if r.get("mem_bytes") is not None]
starts = [r["started_at"] for r in running if r.get("started_at")]
started = max(starts) if starts else None
return {
"kind": "service", "name": s["name"], "stack": stack_of(s["name"]),
"mode": s["mode"],
"image": s.get("image") or next((r["image"] for r in reals if r.get("image")), None),
"running": s["running"], "desired": s["desired"], "state": s["state"],
"trouble": service_trouble(s),
"nodes": sorted({r["host"] for r in reals}, key=str.lower),
"ghost_nodes": sorted({r["host"] for r in s["replicas"] if r["ghost"]}, key=str.lower),
"cpu_cores": sum(cpu) / 100 if cpu else None,
"mem_bytes": sum(mem) if mem else None,
"started_at": started, "up_s": _seconds_up(started, now),
"unhealthy": any(r.get("health") == "unhealthy" for r in reals),
"failures_24h": s.get("failures_24h"), "crash_looping": s.get("crash_looping"),
"last_error": s.get("last_error"), "trouble_since": s.get("trouble_since"),
}
def container_trouble(c, *, stale: bool, exit_failed: bool) -> int | None:
"""How bad a standalone container is, on service_trouble's scale: 1
unhealthy or restarting, 2 died with a failing exit, 3 its report is
stale; None when nothing is wrong (a cleanly stopped one included)."""
if c.health == "unhealthy" or c.status == "restarting":
return 1
if c.status != "running" and exit_failed:
return 2
if stale:
return 3
return None
def container_row(c, *, label: str, host: str, usage: dict, ports: list, stale: bool,
exit_failed: bool, now: datetime) -> dict:
"""A standalone container as a table row; `usage` is service.usage()."""
running = c.status == "running"
return {
"kind": "container", "name": label, "container_name": c.name, "host": host,
"host_id": c.host_id, "stack": c.compose_project or None, "image": c.image,
"status": c.status, "health": c.health, "running": running, "ports": ports,
"usage": usage, "cpu_cores": usage.get("cores") if running else None,
"mem_bytes": usage.get("mem_bytes") if running else None,
"started_at": c.started_at if running else None,
"up_s": _seconds_up(c.started_at, now) if running else None,
"exit_code": c.exit_code, "oom_killed": c.oom_killed, "exit_failed": exit_failed,
"restart_count": c.restart_count or 0, "stale": stale, "scraped_at": c.scraped_at,
"trouble": container_trouble(c, stale=stale, exit_failed=exit_failed),
}
def _hosts(row: dict) -> list[str]:
return row["nodes"] + row["ghost_nodes"] if row["kind"] == "service" else [row["host"]]
def _stopped(row: dict) -> bool:
return (row["running"] == 0) if row["kind"] == "service" else not row["running"]
def filter_rows(rows: list[dict], *, q: str = "", stack: str = "", host: str = "",
state: str = "") -> list[dict]:
"""Rows matching a search (name, container name, image or stack, any
case), a stack (or NO_STACK), a host or node, and a Status filter; an
empty value doesn't filter."""
q = (q or "").strip().lower()
out = []
for r in rows:
if q and not any(q in (v or "").lower() for v in
(r["name"], r.get("container_name"), r["image"], r["stack"])):
continue
if (stack == NO_STACK and r["stack"]) or (stack and stack != NO_STACK and r["stack"] != stack):
continue
if host and host not in _hosts(r):
continue
if state == "problem" and r["trouble"] is None:
continue
if state == "healthy" and (r["trouble"] is not None or _stopped(r)):
continue
if state == "stopped" and not _stopped(r):
continue
out.append(r)
return out
def sort_rows(rows: list[dict], sort: str, desc: bool) -> list[dict]:
"""Rows in a column's order. Ties keep name order; a row with no value for
the column goes last whichever way it runs. "status" is problems first,
worst first, then running, then stopped."""
by_name = sorted(rows, key=lambda r: r["name"].lower())
if sort == "name":
return list(reversed(by_name)) if desc else by_name
if sort == "status":
return sorted(by_name, key=lambda r: (4 if r["trouble"] is None else r["trouble"],
_stopped(r)), reverse=desc)
value = {"stack": lambda r: (r["stack"] or "").lower() or None,
"cpu": lambda r: r["cpu_cores"], "mem": lambda r: r["mem_bytes"],
"up": lambda r: r["up_s"]}[sort]
have = [r for r in by_name if value(r) is not None]
return sorted(have, key=value, reverse=desc) + [r for r in by_name if value(r) is None]
def filter_options(services: list[dict], containers: list[dict]) -> dict:
"""The stack and host choices the filter bar offers, from every row."""
rows = services + containers
return {"stacks": sorted({r["stack"] for r in rows if r["stack"]}, key=str.lower),
"hosts": sorted({h for r in rows for h in _hosts(r)}, key=str.lower)}
plugins/docker/retention.py
+1 -33
View File
@@ -38,8 +38,6 @@ async def run_docker_retention(
events_days: int,
metrics_raw_days: int,
metrics_rollup_days: int,
logs_retention_days: int = 3,
logs_max_bytes_per_container: int = 5_000_000,
now: datetime | None = None,
) -> dict:
"""Roll up + prune Docker time-series. Returns a counts dict for logging.
@@ -49,22 +47,18 @@ async def run_docker_retention(
re-run is idempotent, then delete those raw rows.
2. Prune rolled-up rows older than the rollup window.
3. Prune docker_events older than the events window.
4. Prune docker_logs with a per-container size+age ring (m79): drop lines
older than the age window, then keep only the newest ~cap bytes per
(host, container).
"""
from datetime import timezone
from sqlalchemy import delete, func, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from .models import DockerEvent, DockerLog, DockerMetric, DockerMetricHourly
from .models import DockerEvent, DockerMetric, DockerMetricHourly
if now is None:
now = datetime.now(timezone.utc)
rolled = rolled_rows = events_pruned = rollup_pruned = 0
logs_age_pruned = logs_size_pruned = 0
# ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
raw_cutoff = _rollup_cutoff(now, metrics_raw_days)
@@ -126,35 +120,9 @@ async def run_docker_retention(
)
events_pruned = res.rowcount or 0
# ── 4. Container-log ring: age cutoff, then per-container byte cap (m79) ──
logs_cutoff = now - timedelta(days=logs_retention_days)
res = await session.execute(
delete(DockerLog).where(DockerLog.ts < logs_cutoff)
)
logs_age_pruned = res.rowcount or 0
# Size ring: per (host, container), sum line bytes newest-first; delete a row
# once its strictly-newer siblings already fill the cap. Using the EXCLUSIVE
# prefix (running total minus this row) means the newest row always survives,
# so a single line larger than the cap is never wiped out.
running = func.sum(func.length(DockerLog.line)).over(
partition_by=[DockerLog.host_id, DockerLog.container_name],
order_by=[DockerLog.ts.desc(), DockerLog.id.desc()],
)
prefix_excl = (running - func.length(DockerLog.line)).label("prefix_excl")
ranked = select(DockerLog.id, prefix_excl).subquery()
over_cap = select(ranked.c.id).where(
ranked.c.prefix_excl >= logs_max_bytes_per_container)
res = await session.execute(
delete(DockerLog).where(DockerLog.id.in_(over_cap))
)
logs_size_pruned = res.rowcount or 0
return {
"buckets_rolled": rolled,
"raw_rows_rolled": rolled_rows,
"rollup_pruned": rollup_pruned,
"events_pruned": events_pruned,
"logs_age_pruned": logs_age_pruned,
"logs_size_pruned": logs_size_pruned,
}
plugins/docker/routes.py
+282 -672
View File
File diff suppressed because it is too large Load Diff
plugins/docker/service.py
-675
View File
@@ -1,675 +0,0 @@
"""Docker queries shared by the plugin's pages and its MCP tools.
Each function takes an open session and returns rows or plain data, with no
Quart request context, so the routes and the /mcp tools (which run outside
Quart's pipeline) call the same code.
"""
from __future__ import annotations
import re
from datetime import datetime, timedelta, timezone
from sqlalchemy import func, select
from steward.core.time_range import bucket_seconds
from steward.models.hosts import Host
from .dedup import dedup_by_container_id
from .models import (
DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly, DockerSwarmService,
)
from .swarm_view import replaced_tasks
CRASH_LOOP_DEATHS = 3 # default; Settings → Thresholds sets it
def crash_loop_deaths(config) -> int:
"""Deaths within an hour that make a container or a swarm service a crash
loop, from Settings → Thresholds (app.config["THRESHOLDS"])."""
th = (config.get("THRESHOLDS") or {}).get("crash_loop") or {}
return int(th.get("warn") or CRASH_LOOP_DEATHS)
# Exit codes a deliberate stop leaves: SIGTERM (143), and SIGKILL (137) when
# the process outlives the stop timeout. Without an OOM kill they are someone
# stopping the container, not the container failing.
STOP_EXITS = (137, 143)
def exit_is_failure(exit_code, oom_killed: bool = False) -> bool:
"""Whether a container's exit is a failure: an OOM kill, or a non-zero
exit other than a stop's."""
return bool(oom_killed) or exit_code not in (None, 0, *STOP_EXITS)
def _event_exit(detail) -> int | None:
"""The exit code in a die event's detail ("exit 1"), or None."""
text = (detail or "").strip()
if text.startswith("exit "):
try:
return int(text[5:])
except ValueError:
return None
return None
def event_is_failure(e) -> bool:
"""Whether a die/oom event is a failure. An oom always is; a die is unless
its exit is a stop's (or clean)."""
if e.event == "oom":
return True
return e.event == "die" and exit_is_failure(_event_exit(e.detail))
def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
"""What failed in the last 24h, newest first, for the dashboard count and
the Docker page's list of it: one row per plain container that died or was
OOM-killed, and one per swarm service with failed tasks.
`events` are die/oom events of the last 24h. A swarm task's container dies
whenever Swarm replaces it, so a service with Swarm's task records counts
its failed tasks instead of its containers' deaths; a service without them
(an agent older than 1.8.0) falls back to its tasks' deaths.
Each row: {kind: "container"|"service", name, host_id, host, count, at,
detail, task_id}; task_id is the failed swarm task, so its logs can be
found (None for a container, or a service without task records).
"""
by_prefix = {s["name"] + ".": s for s in swarm_services}
rows: dict[tuple, dict] = {}
# Each manager reports every swarm task, so one task's death can arrive
# once per manager: a service counts each dead task container once.
task_deaths: dict[tuple, set] = {}
for e in sorted(events, key=lambda e: e.at, reverse=True):
if not event_is_failure(e):
continue
svc = next((s for p, s in by_prefix.items() if e.container_name.startswith(p)), None)
if svc is not None and svc["tasks_known"]:
continue
kind, name = ("service", svc["name"]) if svc else ("container", e.container_name)
key = (kind, name) if svc else (kind, e.host_id, name)
if svc is not None:
seen = task_deaths.setdefault(key, set())
if e.container_name in seen:
continue
seen.add(e.container_name)
row = rows.get(key)
if row is None:
rows[key] = {"kind": kind, "name": name, "host_id": e.host_id,
"host": host_name.get(e.host_id, e.host_id), "count": 1, "at": e.at,
"detail": "OOM-killed" if e.event == "oom" else e.detail,
"task_id": (task_of(svc["name"], e.container_name) or (None, None))[1]
if svc else None}
else:
row["count"] += 1
from .swarm_view import task_time
for s in swarm_services:
if s["tasks_known"] and s["failures_24h"]:
err = s["last_error"] or {}
rows[("service", s["name"])] = {
"kind": "service", "name": s["name"], "host_id": None, "host": err.get("node"),
"count": s["failures_24h"], "at": task_time(err) if err else None,
"detail": err.get("error") or err.get("message") or err.get("state"),
"task_id": err.get("task_id")}
epoch = datetime.min.replace(tzinfo=timezone.utc)
return sorted(rows.values(), key=lambda r: r["at"] or epoch, reverse=True)
def task_label(c, node_hostname: dict | None = None) -> str:
"""How a container reads to a person. A swarm task is its service and
slot: "codex_db · 1" for a replica, "agent · node-3" for a global
service's task on a node. Anything else as display_name reads it."""
from .swarm_view import _slot_key
svc = getattr(c, "service_name", None)
if not svc:
return display_name(c)
slot = _slot_key(c)
if slot.isdigit():
return f"{svc} · {slot}"
host = (node_hostname or {}).get(slot) or (node_hostname or {}).get(getattr(c, "node_id", None))
return f"{svc} · {host}" if host else svc
_CI_JOB = re.compile(
r"^(?:GITEA|FORGEJO)-ACTIONS-TASK-\d+-WORKFLOW-(?P<workflow>.+?)-JOB-(?P<job>.+?)(?:-[0-9a-f]{16,})?$")
# docker buildx's container driver names its node "buildx_buildkit_<builder><n>".
_BUILDKIT = re.compile(r"^buildx_buildkit_(?P<builder>.+?)\d*$")
def display_name(c) -> str:
"""A container's name as a list can show it in one short line (#5253): a
swarm task by its service (the host says where it runs), a Gitea or
Forgejo Actions job container as "CI: <workflow> · <job>", a buildx
builder as "BuildKit builder · <name>", anything else by its own name."""
svc = getattr(c, "service_name", None)
if svc:
return svc
m = _CI_JOB.match(c.name or "")
if m:
return f"CI: {m['workflow']} · {m['job']}"
m = _BUILDKIT.match(c.name or "")
if m:
return f"BuildKit builder · {m['builder']}"
return c.name
def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None) -> dict:
"""A container's CPU and memory in terms a reader can weigh.
Docker's CPU% is per core (100% = one core busy), so it is shown as cores
and graded as a share of the host's CPUs when the host's core count is
known. Docker's memory limit is the host's RAM when none is set, so memory
is shown in bytes, against a limit only when one is actually set; mem_pct
is then a share of that limit, otherwise of the host's memory. `cpu_pct`
replaces the container's last sample (an average, say) when given.
"""
cpu = getattr(c, "cpu_pct", None) if cpu_pct is None else cpu_pct
limit = getattr(c, "mem_limit_bytes", None)
limited = bool(limit and mem_total and limit < mem_total * 0.95)
return {
"cores": None if cpu is None else cpu / 100.0,
"cpu_share": None if cpu is None or not cpus else cpu / cpus,
"cpus": cpus,
"mem_bytes": getattr(c, "mem_usage_bytes", None),
"mem_limit": limit if limited else None,
"mem_pct": getattr(c, "mem_pct", None),
"mem_of": "its limit" if limited else ("the host's memory" if mem_total else None),
}
async def host_capacity(db, host_ids) -> dict[str, tuple[int | None, int | None]]:
"""{host_id: (CPU cores, memory bytes)} from the host agent's latest
readings: cores counted from its per-core CPU series, memory from
available ÷ (1 − used%). (None, None) for a host without an agent."""
from sqlalchemy import or_
from steward.models.metrics import PluginMetricLatest
hosts = await host_map(db, set(host_ids))
if not hosts:
return {}
names = {h.name: hid for hid, h in hosts.items()}
rows = (await db.execute(
select(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name,
PluginMetricLatest.value)
.where(PluginMetricLatest.source_module == "host_agent",
or_(PluginMetricLatest.resource_name.in_(list(names)),
*(PluginMetricLatest.resource_name.like(escape_like(n) + ":core%", escape="\\")
for n in names)))
)).all()
cores: dict[str, int] = {}
mem: dict[str, dict] = {}
for res, metric, value in rows:
if res in names:
mem.setdefault(res, {})[metric] = value
elif metric == "cpu_pct":
host = res.rsplit(":core", 1)[0]
cores[host] = cores.get(host, 0) + 1
out = {}
for name, hid in names.items():
m = mem.get(name, {})
avail, used = m.get("mem_available_bytes"), m.get("mem_used_pct")
total = int(avail / (1 - used / 100)) if avail and used is not None and used < 99.5 else None
out[hid] = (cores.get(name), total)
return out
async def host_map(db, host_ids: set[str]) -> dict[str, Host]:
if not host_ids:
return {}
return {
h.id: h for h in (await db.execute(
select(Host).where(Host.id.in_(host_ids)))).scalars().all()
}
def escape_like(text: str) -> str:
"""Make `text` match literally inside a LIKE/ILIKE pattern (escape char \\)."""
return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
async def list_containers(db, host_id: str | None = None) -> list[DockerContainer]:
"""Containers, running first then by name; one host's when host_id is given."""
stmt = select(DockerContainer)
if host_id is not None:
stmt = stmt.where(DockerContainer.host_id == host_id)
stmt = stmt.order_by((DockerContainer.status == "running").desc(), DockerContainer.name)
return list((await db.execute(stmt)).scalars())
async def container_events(db, host_id: str, name: str, limit: int = 50) -> list[DockerEvent]:
"""A container's lifecycle events (start/stop/die/oom/health), newest first."""
return list((await db.execute(
select(DockerEvent)
.where(DockerEvent.host_id == host_id)
.where(DockerEvent.container_name == name)
.order_by(DockerEvent.at.desc())
.limit(limit)
)).scalars())
async def query_logs(db, host_id: str, name: str, stream: str, query: str,
limit: int) -> list:
"""The recent retained lines for one container, newest-first, optionally
filtered by stream and a case-insensitive substring.
Newest-first is deliberate: the viewer replaces the list on each poll, which
resets scroll to the top — so the latest lines stay visible without any
scroll handling, and older lines are a scroll away.
"""
stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line)
.where(DockerLog.host_id == host_id)
.where(DockerLog.container_name == name))
if stream in ("stdout", "stderr"):
stmt = stmt.where(DockerLog.stream == stream)
if query:
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\"))
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
return (await db.execute(stmt)).all()
CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes")
async def container_points(db, host_id: str, name: str, metric: str, since,
target: int = 120) -> list[tuple]:
"""(time, average) points of one container metric since `since`, oldest first.
Raw samples are bucket-averaged in SQL; the part of the range older than
the oldest raw sample (already rolled up and pruned by retention) comes
from the hourly table.
"""
if metric not in CONTAINER_SERIES:
raise ValueError(f"metric must be one of {', '.join(CONTAINER_SERIES)}")
raw_col = getattr(DockerMetric, metric)
width_s = bucket_seconds(since, target)
rbucket = func.date_bin(
func.make_interval(0, 0, 0, 0, 0, 0, width_s), DockerMetric.scraped_at, func.to_timestamp(0),
).label("bucket")
raw = (await db.execute(
select(rbucket, func.avg(raw_col))
.where(DockerMetric.host_id == host_id, DockerMetric.container_name == name,
DockerMetric.scraped_at >= since)
.group_by(rbucket).order_by(rbucket)
)).all()
oldest_raw = raw[0][0] if raw else None
hourly_stmt = (
select(DockerMetricHourly.bucket, getattr(DockerMetricHourly, metric))
.where(DockerMetricHourly.host_id == host_id,
DockerMetricHourly.container_name == name,
DockerMetricHourly.bucket >= since)
.order_by(DockerMetricHourly.bucket)
)
if oldest_raw is not None:
hourly_stmt = hourly_stmt.where(DockerMetricHourly.bucket < oldest_raw)
hourly = (await db.execute(hourly_stmt)).all()
return [(t, float(v)) for t, v in hourly] + [(t, float(v)) for t, v in raw]
# Busiest Containers bars (#5214): the recent past in BARS equal slices.
BARS = 24
def bar_slots(now: datetime, hours: int, bars: int = BARS) -> tuple[list[datetime], int]:
"""The start of each of the last `bars` buckets covering `hours`, oldest
first, aligned as date_bin aligns them (to the epoch), and their width in
seconds. The newest bucket is the one `now` falls in."""
width = max(60, hours * 3600 // bars)
last = int(now.timestamp()) // width * width
return [datetime.fromtimestamp(last - (bars - 1 - i) * width, timezone.utc)
for i in range(bars)], width
async def recent_usage(db, keys, since: datetime, width: int) -> dict:
"""{(host_id, container_name): {bucket start: (cpu_pct, mem_usage_bytes)}}
averaged per bucket since `since`, for the given containers, in one query."""
keys = set(keys)
if not keys:
return {}
bucket = func.date_bin(
func.make_interval(0, 0, 0, 0, 0, 0, int(width)), DockerMetric.scraped_at, func.to_timestamp(0),
).label("bucket")
rows = (await db.execute(
select(DockerMetric.host_id, DockerMetric.container_name, bucket,
func.avg(DockerMetric.cpu_pct), func.avg(DockerMetric.mem_usage_bytes))
.where(DockerMetric.scraped_at >= since,
DockerMetric.host_id.in_({h for h, _ in keys}),
DockerMetric.container_name.in_({n for _, n in keys}))
.group_by(DockerMetric.host_id, DockerMetric.container_name, bucket)
)).all()
out: dict = {}
for host_id, name, at, cpu, mem in rows:
if (host_id, name) in keys:
out.setdefault((host_id, name), {})[at] = (
None if cpu is None else float(cpu), None if mem is None else float(mem))
return out
def scaled_bars(values: list, floor: float) -> dict:
"""Bar heights in % of a roof: the row's own peak, but never below `floor`,
so a container idling at 0.05 cores draws flat and its 0.05 → 0.10 wobble
doesn't look like a 1.5 → 3 core spike. None (no samples) stays a gap."""
present = [v for v in values if v is not None]
peak = max(present) if present else None
roof = max(peak or 0.0, floor)
return {"bars": [None if v is None else round(100.0 * v / roof, 1) for v in values],
"peak": peak, "roof": roof}
async def search_logs(db, *, since, host_id: str | None = None, name: str | None = None,
text: str = "", stream: str = "", before=None, limit: int = 50) -> list:
"""Retained log lines matching a filter, newest first (limit + 1 fetched by callers
that need to know whether more exist)."""
stmt = (select(DockerLog.id, DockerLog.host_id, DockerLog.container_name,
DockerLog.ts, DockerLog.stream, DockerLog.line)
.where(DockerLog.ts >= since))
if host_id:
stmt = stmt.where(DockerLog.host_id == host_id)
if name:
stmt = stmt.where(DockerLog.container_name == name)
if stream in ("stdout", "stderr"):
stmt = stmt.where(DockerLog.stream == stream)
if text:
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(text)}%", escape="\\"))
if before is not None:
stmt = stmt.where(DockerLog.ts < before)
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
return list((await db.execute(stmt)).all())
async def log_context(db, host_id: str, name: str, ts, n: int) -> tuple[list, list]:
"""Up to n lines either side of `ts` in one container's log, each oldest first."""
cols = (DockerLog.id, DockerLog.ts, DockerLog.stream, DockerLog.line)
base = select(*cols).where(DockerLog.host_id == host_id, DockerLog.container_name == name)
before = (await db.execute(
base.where(DockerLog.ts < ts).order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(n)
)).all()
after = (await db.execute(
base.where(DockerLog.ts > ts).order_by(DockerLog.ts, DockerLog.id).limit(n)
)).all()
return list(reversed(before)), list(after)
async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]:
"""{(host_id, container_name): [die/oom events since `since`, newest first]},
leaving out a die that was a stop (event_is_failure)."""
rows = (await db.execute(
select(DockerEvent)
.where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
.order_by(DockerEvent.at.desc())
)).scalars()
out: dict[tuple[str, str], list] = {}
for e in rows:
if event_is_failure(e):
out.setdefault((e.host_id, e.container_name), []).append(e)
return out
async def failure_events(db, since) -> list:
"""Die and OOM events since `since`, newest first; failures_24h() decides
which of them are failures."""
return list((await db.execute(
select(DockerEvent)
.where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
.order_by(DockerEvent.at.desc())
)).scalars())
async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None,
limit: int = 20) -> list[tuple[str, str, datetime]]:
"""(host_id, container_name, last seen) of containers Steward still holds
logs or events for, newest first — including ones Docker has removed.
exact: the name itself, from both logs and events. Otherwise names that
contain `want` (case-insensitive), from events only: a substring match
over every retained log line's name would scan the log table, and a
container Steward watched die has a die or stop event.
"""
seen: dict[tuple[str, str], datetime] = {}
sources = ((DockerLog, DockerLog.ts), (DockerEvent, DockerEvent.at)) if exact \
else ((DockerEvent, DockerEvent.at),)
for model, at in sources:
match = (model.container_name == want if exact
else model.container_name.ilike(f"%{escape_like(want)}%", escape="\\"))
stmt = (select(model.host_id, model.container_name, func.max(at))
.where(match).group_by(model.host_id, model.container_name))
if host_id:
stmt = stmt.where(model.host_id == host_id)
for hid, name, last in (await db.execute(stmt)).all():
key = (hid, name)
if key not in seen or last > seen[key]:
seen[key] = last
rows = sorted(((h, n, t) for (h, n), t in seen.items()), key=lambda r: r[2], reverse=True)
return rows[:limit]
# A swarm task's container is named "<service>.<slot>.<task id>": slot is the
# replica number, or the node id for a global service; task ids are 25 chars.
_TASK_SUFFIX = re.compile(r"^(?P<slot>[A-Za-z0-9]+)\.(?P<task>[a-z0-9]{25})$")
def task_of(service: str, container_name: str) -> tuple[str, str] | None:
"""(slot, task id) when `container_name` is one of `service`'s task
containers, else None — so "arr" does not claim "arr_radarr.1.…"."""
if not container_name.startswith(service + "."):
return None
m = _TASK_SUFFIX.match(container_name[len(service) + 1:])
return (m["slot"], m["task"]) if m else None
async def service_containers(db, service: str) -> list[tuple[str, str]]:
"""(host_id, container_name) of every task container of a swarm service
that Steward still holds anything for: the live ones, and the ones Docker
removed that left events (each task container has a start event). The
page's logs and history read across all of them (#5302)."""
pattern = escape_like(service + ".") + "%"
found: set[tuple[str, str]] = set()
for model, col in ((DockerContainer, DockerContainer.name),
(DockerEvent, DockerEvent.container_name)):
rows = (await db.execute(
select(model.host_id, col).where(col.like(pattern, escape="\\")).distinct())).all()
found.update((h, n) for h, n in rows if task_of(service, n))
return sorted(found)
async def query_service_logs(db, containers, stream: str, query: str, limit: int,
task: str | None = None) -> list:
"""The newest retained lines across a service's task containers, newest
first, each with its container name so the page can say which task wrote
it. `task` keeps one task's lines (a failure's "Logs →")."""
from sqlalchemy import tuple_
pairs = [(h, n) for h, n in containers if task is None or n.endswith("." + task)]
if not pairs:
return []
stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line, DockerLog.container_name)
.where(tuple_(DockerLog.host_id, DockerLog.container_name).in_(pairs)))
if stream in ("stdout", "stderr"):
stmt = stmt.where(DockerLog.stream == stream)
if query:
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\"))
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
return (await db.execute(stmt)).all()
def sum_series(samples, since: datetime, until: datetime, points: int = 60) -> list:
"""[(bucket start, cpu %, memory bytes)] summed across containers: each
container's samples are averaged within a bucket, then the containers are
added, so two replicas read as their combined load. `samples` are
(container_name, at, cpu_pct, mem_bytes); buckets with no sample are left
out rather than read as zero."""
width = max((until - since).total_seconds() / points, 60.0)
acc: dict[int, dict[str, list]] = {}
for name, at, cpu, mem in samples:
i = int((at - since).total_seconds() // width)
if i < 0:
continue
per = acc.setdefault(i, {}).setdefault(name, [0.0, 0, 0])
per[0] += cpu or 0.0
per[1] += mem or 0
per[2] += 1
out = []
for i in sorted(acc):
cpu = sum(c / n for c, _, n in acc[i].values())
mem = sum(m / n for _, m, n in acc[i].values())
out.append((since + timedelta(seconds=i * width), cpu, int(mem)))
return out
async def service_history(db, containers, since: datetime) -> list:
"""sum_series over a service's task containers' samples since `since`."""
from sqlalchemy import tuple_
if not containers:
return []
rows = (await db.execute(
select(DockerMetric.container_name, DockerMetric.scraped_at,
DockerMetric.cpu_pct, DockerMetric.mem_usage_bytes)
.where(tuple_(DockerMetric.host_id, DockerMetric.container_name).in_(list(containers)),
DockerMetric.scraped_at >= since)
.order_by(DockerMetric.scraped_at))).all()
return sum_series(rows, since, datetime.now(timezone.utc))
async def unhealthy_since(db, keys) -> dict[tuple[str, str], datetime]:
"""{(host_id, container_name): when it last turned unhealthy} for the given
containers, from their health_change events; absent when none is retained."""
keys = set(keys)
if not keys:
return {}
rows = (await db.execute(
select(DockerEvent)
.where(DockerEvent.event == "health_change",
DockerEvent.detail.like("%→unhealthy"),
DockerEvent.container_name.in_({name for _, name in keys}))
.order_by(DockerEvent.at.desc())
)).scalars()
out: dict[tuple[str, str], datetime] = {}
for e in rows:
key = (e.host_id, e.container_name)
if key in keys and key not in out:
out[key] = e.at
return out
PRUNE_TARGETS = ("containers", "images", "system")
def prune_extra_vars(prune_target: str) -> dict:
"""Extra-vars for the bundled docker_prune playbook. "Prune unused images"
means ALL unused (docker image prune -a), not just dangling; the full system
prune stays conservative (-f, no -a) per the operator's choice (m78)."""
extra_vars: dict = {"prune_target": prune_target}
if prune_target == "images":
extra_vars["prune_all_images"] = True
return extra_vars
# ── what is wrong with a container ───────────────────────────────────────────
# Shared by the Docker page's "Needs attention" and the MCP tools, so both
# rank the same containers the same way.
PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
def container_problems(c, failures_1h: int, failed_24h: bool,
crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
"""What is wrong with one container, most severe first; [] when nothing is."""
status = (c.status or "").lower()
out = []
if status == "restarting":
out.append("restarting")
if failures_1h >= crash_loop_deaths:
out.append("crash_looping")
if status == "running" and (c.health or "") == "unhealthy":
out.append("unhealthy")
if c.oom_killed:
out.append("oom_killed")
if status in ("exited", "dead") and exit_is_failure(c.exit_code) and failed_24h:
out.append("exited_nonzero")
return out
def problem_since(problems: list[str], deaths: list, hour_ago: datetime,
unhealthy_at: datetime | None = None) -> datetime | None:
"""When a container's worst problem began; None when the retained events
cannot say. `deaths` are its die/oom events of the last 24h, newest first.
A restart or crash loop dates from its first death in the last hour, an
OOM kill or error exit from the death that ended it, and a failing
healthcheck from when its health last turned unhealthy.
"""
if not problems:
return None
worst = problems[0]
if worst in ("restarting", "crash_looping"):
in_hour = [e.at for e in deaths if e.at >= hour_ago]
return min(in_hour) if in_hour else (deaths[0].at if deaths else None)
if worst == "unhealthy":
return unhealthy_at
if worst == "oom_killed":
ooms = [e.at for e in deaths if e.event == "oom"]
return ooms[0] if ooms else (deaths[0].at if deaths else None)
return deaths[0].at if deaths else None
PROBLEM_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
"oom_killed": "warning", "exited_nonzero": "warning"}
PROBLEM_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
"unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
"exited_nonzero": "exited with an error"}
async def ranked_containers(db, now: datetime, crash_loop_deaths: int) -> list[dict]:
"""Every container (deduplicated) with what is wrong with it, most severe
first: {container, problems, since, replaced}. A swarm task Swarm has
replaced has no problems; its failure belongs to its service."""
containers = dedup_by_container_id(await list_containers(db))
services = list((await db.execute(select(DockerSwarmService))).scalars())
replaced = replaced_tasks(containers, services)
failures = await recent_failures_by_container(db, now - timedelta(hours=24))
hour_ago = now - timedelta(hours=1)
sick = await unhealthy_since(
db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"])
out = []
for c in containers:
key = (c.host_id, c.name)
gone = key in replaced
events = failures.get(key, [])
recent = sum(1 for e in events if e.at >= hour_ago)
problems = [] if gone else container_problems(c, recent, bool(events), crash_loop_deaths)
out.append({"container": c, "problems": problems, "replaced": gone,
"since": problem_since(problems, events, hour_ago, sick.get(key))})
return out
def needs_attention(ranked: list[dict]) -> list[dict]:
"""The containers with something wrong, worst problem first, then by host
and name."""
return sorted((r for r in ranked if r["problems"]),
key=lambda r: (PROBLEMS.index(r["problems"][0]), r["container"].host_id, r["container"].name))
def stale_after_seconds(config) -> int:
"""How old a container's last report may be before the page calls it
stale: the host agent's own staleness window, since the agent reports it."""
return int(((config.get("PLUGINS") or {}).get("host_agent") or {}).get("stale_after_seconds", 180))
async def disk_hosts(db, since: datetime, *, points: int = 60) -> list[dict]:
"""Each Docker host's disk as `docker system df` last said, most space
first, with its recorded total since `since` (#5304): the points, oldest
first, and the growth over them (disk_view.growth; None until there are
two). Hosts are named by their Steward host."""
from steward.core import metric_series
from .disk_view import DISK_PARTS, growth
from .models import DockerDiskUsage
summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
hosts = await host_map(db, {s.host_id for s in summaries})
out = []
for s in summaries:
name = hosts[s.host_id].name if s.host_id in hosts else s.host_id
parts = {m: int(getattr(s, f) or 0) for m, f in DISK_PARTS.items()}
series = await metric_series.points(db, "docker", name, "disk_total_bytes", since, target=points)
out.append({"host_id": s.host_id, "host_name": name, "as_of": s.updated_at,
"total": sum(parts.values()), "parts": parts,
"reclaimable": int(s.images_reclaimable or 0), "points": series,
"growth": growth(series)})
return sorted(out, key=lambda h: (-h["total"], h["host_name"].lower()))
plugins/docker/swarm_nodes.py
-71
View File
@@ -1,71 +0,0 @@
# plugins/docker/swarm_nodes.py
"""The Swarm page laid out by node (#5303): one card per node with its role,
whether it takes work, what its containers use against what the machine has,
and the services Swarm placed on it. Model-free, so it is unit-tested without
a database; routes.swarm() feeds it.
A node with a Steward agent reports its own containers, so its services carry
CPU and memory. A node without one is known only from the managers' placement:
its services are listed by count, with no figures.
"""
from __future__ import annotations
from .swarm_view import _placement
def node_ok(n) -> bool:
"""A node that is up and taking work."""
return n.status == "ready" and n.availability == "active"
def node_cards(nodes, services, containers, *, node_host: dict, host_names: dict,
capacity: dict, service_state: dict) -> list[dict]:
"""One card per node, a node in trouble first, then managers, then by name.
`nodes` and `services` are one swarm's deduplicated rows; `containers`
every container row (only running swarm tasks on these nodes count);
`node_host` maps a node id to the Steward host on it, `capacity` a host id
to (cores, memory bytes), `service_state` a service name to healthy,
degraded or down."""
running_on: dict[str, dict[str, dict]] = {}
for c in containers:
if not (c.node_id and c.service_name and c.status == "running"):
continue
row = running_on.setdefault(c.node_id, {}).setdefault(
c.service_name, {"name": c.service_name, "tasks": 0, "cpu_pct": 0.0, "mem_bytes": 0,
"measured": True})
row["tasks"] += 1
row["cpu_pct"] += c.cpu_pct or 0.0
row["mem_bytes"] += c.mem_usage_bytes or 0
placed: dict[str, dict[str, int]] = {}
for s in services:
for p in _placement(s):
count = int(p.get("running") or 0)
if count > 0 and p.get("node_id"):
placed.setdefault(p["node_id"], {})[s.service_name] = count
cards = []
for n in nodes:
rows = dict(running_on.get(n.node_id, {}))
for name, count in placed.get(n.node_id, {}).items():
rows.setdefault(name, {"name": name, "tasks": count, "cpu_pct": None, "mem_bytes": None,
"measured": False})
for r in rows.values():
r["state"] = service_state.get(r["name"], "healthy")
r["cores"] = r["cpu_pct"] / 100 if r["cpu_pct"] is not None else None
tasks = sorted(rows.values(), key=lambda r: (r["state"] == "healthy", -(r["cores"] or 0.0),
r["name"].lower()))
host_id = node_host.get(n.node_id)
cores, mem = capacity.get(host_id, (None, None)) if host_id else (None, None)
measured = [r for r in tasks if r["measured"]]
cards.append({
"node_id": n.node_id, "hostname": n.hostname or n.node_id[:12], "role": n.role,
"leader": n.leader, "status": n.status, "availability": n.availability, "ok": node_ok(n),
"host_id": host_id, "host_name": host_names.get(host_id) if host_id else None,
"agent": bool(running_on.get(n.node_id)) or host_id is not None,
"tasks": tasks, "task_count": sum(r["tasks"] for r in tasks),
"cores_used": sum(r["cores"] for r in measured) if measured else None,
"mem_used": sum(r["mem_bytes"] for r in measured) if measured else None,
"cores": cores, "mem": mem,
})
return sorted(cards, key=lambda c: (c["ok"], c["role"] != "manager", c["hostname"].lower()))
plugins/docker/swarm_view.py
+14 -262
View File
@@ -11,28 +11,12 @@ For every swarm service we list the real replicas we have detail for, then add
"ghost" replicas for the remaining placement count on each node (agent-less
nodes). Non-swarm containers pass through grouped by host, unchanged.
A third source, DockerSwarmService.tasks_json (agent ≥ 1.8.0), is Swarm's own
record of each service's newest tasks. It decides which task containers are
*superseded* — replaced by Swarm, so neither a replica nor a crash — and gives
each service its failures across task retries and the error of its last
failed task. Swarm restarts a task by creating a new one (a new container), so
failures can only be counted per service, never per container.
Kept model-free (no ORM imports) so it's unit-testable and safe to import in
tests without the plugin-loader "table already defined" gotcha.
"""
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
# Task states that mean the task ended in error: `failed` (its container exited
# non-zero or never started) and `rejected` (Swarm could not start it at all,
# e.g. no such image, a bad mount).
FAILED_TASK_STATES = ("failed", "rejected")
# Desired states of a task Swarm still means to run. Anything else (shutdown,
# remove, complete, orphaned) is a task Swarm has finished with.
_CURRENT_DESIRED = ("running", "ready", "")
def _service_state(running: int, desired: int) -> str:
@@ -52,222 +36,30 @@ def _placement(service) -> list[dict]:
return [p for p in data if isinstance(p, dict)]
def parse_tasks(service) -> list[dict]:
"""A service row's task records, newest first; [] when its agent sent none."""
raw = getattr(service, "tasks_json", None) or "[]"
try:
data = json.loads(raw)
except (ValueError, TypeError):
return []
return [t for t in data if isinstance(t, dict)]
def build_swarm_services(containers, services, node_hostname: dict, host_name: dict) -> dict:
"""Return {"services": [...], "standalone": [...]}.
def task_time(task: dict) -> datetime | None:
"""A task's status timestamp. Docker writes nanoseconds; Python ≥ 3.11 keeps
the first six digits."""
raw = task.get("at") or ""
try:
at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
except ValueError:
return None
return at if at.tzinfo else at.replace(tzinfo=timezone.utc)
def task_failures(tasks: list[dict], since: datetime) -> int:
"""How many tasks ended in error at or after `since`."""
n = 0
for t in tasks:
at = task_time(t)
if t.get("state") in FAILED_TASK_STATES and at is not None and at >= since:
n += 1
return n
def last_task_error(tasks: list[dict]) -> dict | None:
"""Why the service's newest troubled task is in trouble, from Swarm itself.
The newest task that failed or was rejected, or that Swarm attached an error
to while still trying (a pending task with "no suitable node", say). None
when no retained task has a problem.
"""
for t in tasks: # newest first
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
return {"state": t.get("state"), "error": t.get("error") or None,
"message": t.get("message") or None, "exit_code": t.get("exit_code"),
"node_id": t.get("node_id") or None, "at": t.get("at") or None,
"task_id": t.get("task_id") or None, "slot": t.get("slot")}
return None
def task_summary(tasks: list[dict], now: datetime, crash_loop_deaths: int,
node_hostname: dict) -> dict:
"""What a service's task records say: tasks_known, failures_1h,
failures_24h (None without records), crash_looping, and last_error with
the node's hostname as `node`."""
known = bool(tasks)
failures_1h = task_failures(tasks, now - timedelta(hours=1)) if known else None
last_error = last_task_error(tasks)
if last_error is not None:
nid = last_error["node_id"]
last_error["node"] = node_hostname.get(nid) or (nid[:12] if nid else None)
at = task_time(last_error)
last_error["age_s"] = max(0, int((now - at).total_seconds())) if at else None
return {
"tasks_known": known,
"failures_1h": failures_1h,
"failures_24h": task_failures(tasks, now - timedelta(hours=24)) if known else None,
"crash_looping": bool(known and failures_1h >= crash_loop_deaths),
"last_error": last_error,
}
def _oldest_failure(tasks: list[dict], since: datetime) -> datetime | None:
times = [at for t in tasks if t.get("state") in FAILED_TASK_STATES
and (at := task_time(t)) is not None and at >= since]
return min(times, default=None)
def trouble_since(tasks: list[dict], state: str, crash_looping: bool,
now: datetime) -> datetime | None:
"""When a service's current trouble began, as far as its retained tasks
tell; None when it has none, or the tasks cannot say.
A crash loop dates from its first failure in the last hour, and a healthy
service that lost tasks from its first failure in the last 24h. A service
short of replicas dates from its troubled slots (replicated: the slot;
global: the node): in each slot whose newest task is not running, the
oldest of its newest troubled tasks (failed, rejected, or carrying an
error), walking back until a running task interrupts them. Swarm keeps few
old tasks, so a long outage reads as the oldest one retained: a lower
bound.
"""
if crash_looping:
return _oldest_failure(tasks, now - timedelta(hours=1))
if state == "healthy":
return _oldest_failure(tasks, now - timedelta(hours=24))
by_slot: dict = {}
for t in tasks: # newest first, so each slot's list is too
by_slot.setdefault(t.get("slot") or t.get("node_id"), []).append(t)
began = []
for slot_tasks in by_slot.values():
oldest = None
for t in slot_tasks:
if t.get("state") == "running":
break
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
oldest = task_time(t) or oldest
if oldest is not None:
began.append(oldest)
return min(began, default=None)
def _slot_key(c) -> str:
"""A task container's slot within its service: "<service>.<slot>.<task>" for
replicated services, "<service>.<node id>.<task>" for global ones."""
svc, name = c.service_name, c.name or ""
rest = name[len(svc) + 1:] if name.startswith(svc + ".") else name
return rest.split(".", 1)[0]
def superseded_containers(swarm_cs, tasks_by_id: dict[str, dict]) -> set[tuple]:
"""(host_id, name) of task containers Swarm has replaced.
With the task on record, its desired state decides: Swarm sets a replaced
or failed task's desired state to shutdown. Without it (an older agent, or a
task older than the retained history), a container that is not running is
superseded when another task in the same slot is running or started later.
"""
out: set[tuple] = set()
by_slot: dict[tuple, list] = {}
for c in swarm_cs:
by_slot.setdefault((c.service_name, _slot_key(c)), []).append(c)
for c in swarm_cs:
task = tasks_by_id.get(getattr(c, "task_id", None) or "")
if task is not None:
if task.get("desired_state") not in _CURRENT_DESIRED:
out.add((c.host_id, c.name))
continue
if (getattr(c, "status", "") or "").lower() == "running":
continue
mine = getattr(c, "started_at", None)
for other in by_slot[(c.service_name, _slot_key(c))]:
if other is c:
continue
theirs = getattr(other, "started_at", None)
if ((getattr(other, "status", "") or "").lower() == "running"
or (theirs is not None and (mine is None or theirs > mine))):
out.add((c.host_id, c.name))
break
return out
def _newest_per_name(services) -> dict[str, object]:
"""Every manager reports the same cluster-global service set; keep the most
recently updated row per name, so a manager on an older agent (no tasks)
doesn't shadow one that reports them."""
by_name: dict[str, object] = {}
for s in services:
name = getattr(s, "service_name", None)
if not name:
continue
kept = by_name.get(name)
if kept is None:
by_name[name] = s
continue
a, b = getattr(s, "updated_at", None), getattr(kept, "updated_at", None)
if a is not None and (b is None or a > b):
by_name[name] = s
return by_name
def replaced_tasks(containers, services) -> set[tuple]:
"""(host_id, name) of the task containers Swarm has replaced, from the
container rows and the managers' service rows. What the container views use
to leave history out of their problem lists."""
swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
tasks_by_id = {t["task_id"]: t for s in _newest_per_name(services).values()
for t in parse_tasks(s) if t.get("task_id")}
return superseded_containers(swarm_cs, tasks_by_id)
def build_swarm_services(containers, services, node_hostname: dict, host_name: dict,
*, now: datetime | None = None, crash_loop_deaths: int = 3) -> dict:
"""Return {"services": [...], "standalone": [...], "superseded": {(host_id, name)}}.
services[i] = {name, mode, image, desired, running, state, replicas:[...],
previous_tasks, tasks_known, failures_1h, failures_24h,
crash_looping, last_error}
services[i] = {name, mode, desired, running, state, replicas:[...]}
real replica = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
mem_bytes, started_at, image, health, restart_count}
# current tasks only
health, restart_count}
ghost replica = {ghost:True, host, count} # placement with no local row
previous_tasks = task containers Swarm replaced, left out of replicas
failures_* = tasks that failed or were rejected in the window (None
without task data); crash_looping = failures_1h ≥ the
Settings → Thresholds crash-loop count
last_error = last_task_error(), plus `node` (the node's hostname)
trouble_since = trouble_since(): when the current trouble began, or None
standalone[i] = {host_id, host, containers:[...]} # non-swarm, by host
superseded = the task containers Swarm replaced, so callers can skip them
"""
now = now or datetime.now(timezone.utc)
containers = list(containers)
swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
standalone_cs = [c for c in containers if not getattr(c, "service_name", None)]
svc_by_name = _newest_per_name(services)
tasks_by_svc = {name: parse_tasks(s) for name, s in svc_by_name.items()}
tasks_by_id = {t["task_id"]: t for ts in tasks_by_svc.values() for t in ts
if t.get("task_id")}
superseded = superseded_containers(swarm_cs, tasks_by_id)
# Dedup service rows by name (every manager reports the same cluster-global
# set); keep the first — placement is identical across managers.
svc_by_name: dict[str, object] = {}
for s in services:
name = getattr(s, "service_name", None)
if name and name not in svc_by_name:
svc_by_name[name] = s
# Current task containers grouped by service name; replaced ones are counted.
# Real containers grouped by service name.
cs_by_service: dict[str, list] = {}
previous: dict[str, int] = {}
for c in swarm_cs:
if (c.host_id, c.name) in superseded:
previous[c.service_name] = previous.get(c.service_name, 0) + 1
continue
cs_by_service.setdefault(c.service_name, []).append(c)
def _host_of(c):
@@ -290,9 +82,6 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
"status": getattr(c, "status", "unknown"),
"cpu_pct": getattr(c, "cpu_pct", None),
"mem_pct": getattr(c, "mem_pct", None),
"mem_bytes": getattr(c, "mem_usage_bytes", None),
"started_at": getattr(c, "started_at", None),
"image": getattr(c, "image", None) or None,
"health": getattr(c, "health", None),
"restart_count": getattr(c, "restart_count", 0) or 0,
})
@@ -328,20 +117,13 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
if desired is None:
desired = len(replicas)
tasks = tasks_by_svc.get(name, [])
state = _service_state(int(running or 0), int(desired or 0))
summary = task_summary(tasks, now, crash_loop_deaths, node_hostname)
out_services.append({
"name": name,
"mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
"image": getattr(svc, "image", None) if svc is not None else None,
"running": running,
"desired": desired,
"state": state,
"state": _service_state(int(running or 0), int(desired or 0)),
"replicas": replicas,
"previous_tasks": previous.get(name, 0),
**summary,
"trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
})
# Non-swarm containers grouped by host, running-first order preserved.
@@ -355,34 +137,4 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
g["containers"].append(c)
standalone = sorted(groups.values(), key=lambda g: g["host"].lower())
return {"services": out_services, "standalone": standalone, "superseded": superseded}
def service_trouble(s: dict) -> int | None:
"""How bad a built service is, for sorting and for Needs attention: 0 a
crash loop, 1 no replica running, 2 short of replicas, 3 healthy but with
failed tasks in the last 24h; None when nothing is wrong."""
if s.get("crash_looping"):
return 0
if s.get("state") == "down":
return 1
if s.get("state") == "degraded":
return 2
if s.get("failures_24h"):
return 3
return None
def services_problems_first(services: list[dict]) -> list[dict]:
"""Services in trouble first, worst first, then the rest by name."""
return sorted(services, key=lambda s: (
4 if service_trouble(s) is None else service_trouble(s), s["name"].lower()))
def failing_services(services: list[dict]) -> list[dict]:
"""The services that belong in Needs attention: a crash loop, or short of
their desired replicas. A healthy service whose failed tasks Swarm has
already replaced is history, shown on the service, not a problem now."""
# A crash loop ranks 0, so test the rank against None, not for truth.
return [s for s in services_problems_first(services)
if (t := service_trouble(s)) is not None and t <= 2]
return {"services": out_services, "standalone": standalone}
plugins/docker/templates/docker/_container_history.html
+17
View File
@@ -0,0 +1,17 @@
{# docker/_container_history.html — CPU/mem sparklines for the selected range #}
{% if have_data %}
<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:1.5rem;">
<div>
<div class="section-title" style="margin-bottom:0.4rem;">CPU %</div>
{{ sparkline_cpu | safe }}
</div>
<div>
<div class="section-title" style="margin-bottom:0.4rem;">Memory %</div>
{{ sparkline_mem | safe }}
</div>
</div>
{% else %}
<div style="color:var(--text-muted);font-size:0.85rem;">
Not enough samples in this range yet — history fills in as the agent reports.
</div>
{% endif %}
plugins/docker/templates/docker/_container_logs_lines.html
-24
View File
@@ -1,24 +0,0 @@
{# docker/_container_logs_lines.html — log-line fragment, HTMX-polled (m79).
{{ l.line }} is auto-escaped by Jinja, so log content can't inject markup.
A service's lines (#5302) carry the task each came from. #}
{% if lines %}
{% for l in lines %}
<div style="display:flex;gap:0.6rem;white-space:pre-wrap;word-break:break-word;">
<span style="color:var(--text-dim);flex-shrink:0;" title="{{ l.ts }}">{{ l.ts.strftime("%m-%d %H:%M:%S") }}</span>
{% if l.task %}<span class="tone-muted" style="flex-shrink:0;white-space:nowrap;" title="Slot and task id">{{ l.task }}</span>{% endif %}
<span style="flex-shrink:0;width:3.2rem;color:{% if l.stream == 'stderr' %}var(--red){% else %}var(--text-muted){% endif %};">{{ l.stream }}</span>
<span style="flex:1;min-width:0;{% if l.stream == 'stderr' %}color:var(--text);{% endif %}">{{ l.line }}</span>
</div>
{% endfor %}
{% else %}
<div style="color:var(--text-muted);padding:1rem 0.25rem;">
{% if filtered %}
No lines match the current filter.
{% else %}
No logs collected yet for <code>{{ name }}</code>. Lines appear within a few
seconds of {{ "one of its tasks" if service else "the container" }} writing to stdout/stderr — unless it's on the log
exclude list or log collection is turned off in
<a href="/settings/thresholds/">Settings → Thresholds &amp; Retention</a>.
{% endif %}
</div>
{% endif %}
plugins/docker/templates/docker/_disk_hosts.html
-37
View File
@@ -1,37 +0,0 @@
{# docker/_disk_hosts.html — every Docker host's disk side by side (#5304):
what it uses now, how much that grew over the selected range, and a line
of it; the parts after. A host is linked to its own section below. #}
{% if rows %}
<table class="table">
<thead><tr>
<th>Host</th>
<th class="num">Docker uses</th>
<th class="num" title="Change in the total over the last {{ range_key }}, from Steward's own record of it">Change ({{ range_key }})</th>
<th>Over {{ range_key }}</th>
<th class="num">Images</th><th class="num">Volumes</th><th class="num" title="Written by containers on top of their images">Container files</th>
<th class="num">Build cache</th><th class="num" title="Held by images no container uses">Reclaimable</th>
</tr></thead>
<tbody>
{% for r in rows %}
{% set g = r.growth %}
<tr>
<td><a href="#host-{{ r.host_id }}">{{ r.host_name }}</a></td>
<td class="num"><strong>{{ r.total|bytes }}</strong></td>
<td class="num">
{%- if g is none %}<span class="tone-dim" title="Steward records this from now on; growth shows once there are two readings">—</span>
{%- else %}<span class="tone-{{ 'warn' if g.pct is not none and g.pct >= 10 else ('ok' if g.change < 0 else 'muted') }}"
title="{{ g.from|bytes }} → {{ g.to|bytes }} since {{ g.since.strftime('%Y-%m-%d %H:%M UTC') }}">{{ "+" if g.change > 0 else ("−" if g.change < 0 else "±") }}{{ (g.change|abs)|bytes }}</span>{% endif -%}
</td>
<td class="tone-info">{{ r.spark|safe }}</td>
<td class="num">{{ r.parts.disk_images_bytes|bytes }}</td>
<td class="num">{{ r.parts.disk_volumes_bytes|bytes }}</td>
<td class="num">{{ r.parts.disk_containers_bytes|bytes }}</td>
<td class="num">{{ r.parts.disk_build_cache_bytes|bytes }}</td>
<td class="num">{% if r.reclaimable %}<span class="tone-warn">{{ r.reclaimable|bytes }}</span>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p class="tone-muted" style="margin:0;padding:1rem;">No host has reported its Docker disk yet.</p>
{% endif %}
plugins/docker/templates/docker/_swarm_failures.html
-25
View File
@@ -1,25 +0,0 @@
{#- One line on a swarm service's task failures, from Swarm's own task records
(agent ≥ 1.8.0): a crash loop or the failures of the last 24h, and how the
newest troubled task ended, in Swarm's words. Expects `s` with state,
crash_looping, failures_1h, failures_24h and last_error; `style` is
optional extra CSS for the wrapper. Renders nothing for a healthy service
with no recent failures. -#}
{%- set e = s.last_error -%}
{%- if s.crash_looping or s.failures_24h or (e and s.state != 'healthy') %}
<div style="display:flex;flex-wrap:wrap;align-items:baseline;gap:0.2rem 0.6rem;font-size:0.78rem;color:var(--text-muted);{{ style or '' }}">
{% if s.crash_looping %}
<span style="color:var(--red);font-weight:500;">Crash-looping · {{ s.failures_1h }} tasks failed in the last hour</span>
{% elif s.failures_24h %}
<span style="color:var(--orange);">{{ s.failures_24h }} task{{ 's' if s.failures_24h != 1 }} failed in the last 24h</span>
{% endif %}
{% if e %}
<span title="How Swarm recorded the end of this service's newest failed task">
Last task {{ e.state }}:
<span style="color:var(--text);font-family:ui-monospace,monospace;overflow-wrap:anywhere;">{{ e.error or e.message or e.state }}</span>
{%- if e.exit_code %} · exit {{ e.exit_code }}{% endif %}
{%- if e.node %} · {{ e.node }}{% endif %}
{%- if e.age_s is not none %} · {{ e.age_s|duration }} ago{% endif %}
</span>
{% endif %}
</div>
{%- endif %}
plugins/docker/templates/docker/_usage.html
-26
View File
@@ -1,26 +0,0 @@
{#- A container's CPU and memory as service.usage() describes them (#5209).
Docker counts CPU per core (100% = one core busy), so CPU reads as cores,
toned by its share of the host's CPUs when the host's core count is known.
Memory reads in bytes, against a limit only when one is actually set;
mem_pct is then a share of that limit, otherwise of the host's memory. -#}
{% macro cores(n) -%}
{{ "%.2f"|format(n) if n < 1 else "%.1f"|format(n) }} {{ "core" if n == 1 else "cores" }}
{%- endmacro %}
{% macro cpu(u) -%}
{%- if u.cores is none -%}<span class="tone-dim">—</span>
{%- else -%}
<span class="tone-{{ status_class(u.cpu_share, 'cpu') }}"
title="Docker counts CPU per core: {{ '%.0f'|format(u.cores * 100) }}% of one core{% if u.cpus %}, {{ '%.1f'|format(u.cpu_share) }}% of this host's {{ u.cpus }} core{{ 's' if u.cpus != 1 }}{% endif %}">{{ cores(u.cores) }}</span>
{%- endif -%}
{%- endmacro %}
{% macro mem(u) -%}
{%- if u.mem_bytes is none -%}<span class="tone-dim">—</span>
{%- else -%}
<span class="tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of else 'dim' }}"
title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% else %}No memory limit set{% endif %}">{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %}</span>
{%- endif -%}
{%- endmacro %}
plugins/docker/templates/docker/_usage_history.html
-23
View File
@@ -1,23 +0,0 @@
{# docker/_usage_history.html — CPU and memory over the selected range, for a
service (summed across its replicas, #5302) or one container (#5310): the
latest and the peak, over a line that spans its half. The line is drawn
against at least cpu_floor / mem_floor, so idling near zero draws flat. #}
{% from "docker/_usage.html" import cores %}
{% if have_data %}
<div class="history-pair">
<div>
<div class="section-title">CPU</div>
<div style="font-variant-numeric:tabular-nums;">{{ cores(cpu_now) }} <span class="tone-dim">now · peak {{ cores(cpu_peak) }}</span></div>
<div class="spark" title="Drawn from zero to its peak, or to {{ cores(cpu_floor)|striptags }} when the peak is lower">{{ spark_cpu|safe }}</div>
</div>
<div>
<div class="section-title">Memory</div>
<div style="font-variant-numeric:tabular-nums;">{{ mem_now|bytes }} <span class="tone-dim">now · peak {{ mem_peak|bytes }}</span></div>
<div class="spark" title="Drawn from zero to its peak, or to {{ mem_floor|bytes }} when the peak is lower">{{ spark_mem|safe }}</div>
</div>
</div>
{% else %}
<p class="tone-muted" style="margin:0;font-size:0.85rem;">
Not enough samples in the last {{ range_key }} yet. History fills in as the agent reports.
</p>
{% endif %}
plugins/docker/templates/docker/container_detail.html
+79 -94
View File
@@ -1,94 +1,78 @@
{# docker/container_detail.html — one container (#5310), shaped like a
service's page: a header that says whether it is well and what it is; why
it stopped, only while it has; what it moved since it started; its CPU and
memory over time (the one home for what it uses now); its lifecycle.
A swarm task is titled by its service and slot, the raw name under it. #}
{# docker/container_detail.html — full detail page for one container #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, status_badge %}
{% block title %}{{ label or name }} — Docker — Steward{% endblock %}
{% block breadcrumb %}
{%- if container is not none and container.service_name -%}
{{ crumbs([("Docker", "/plugins/docker/"), (container.service_name, "/plugins/docker/service/" ~ container.service_name), (label, "")]) }}
{%- else -%}
{{ crumbs([("Docker", "/plugins/docker/"), (label or name, "")]) }}
{%- endif -%}
{% endblock %}
{% from "_macros.html" import crumbs %}
{% block title %}{{ name }} — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
{% block content %}
{% if container is none %}
<div class="card empty-state">
<strong>Container not found</strong>
<span>No container named <code>{{ name }}</code> is reported for this host now. It may have been
removed, or the host agent has not reported recently. <a href="/plugins/docker/">Back to Docker</a></span>
<div class="card" style="text-align:center;padding:3rem;">
<div style="font-weight:600;margin-bottom:0.4rem;">Container not found</div>
<div style="color:var(--text-muted);font-size:0.9rem;">
No container named <code>{{ name }}</code> is currently reported for this host.
It may have been removed, or the host agent hasn't reported recently.
</div>
</div>
{% else %}
{% set c = container %}
{# ── Header: the verdict, then what it is ────────────────────────────────── #}
<div class="page-header">
<div style="min-width:0;">
<h1 class="page-title" style="overflow-wrap:anywhere;">{{ label }}</h1>
<p class="page-subtitle">
{% if c.status == 'running' %}{{ status_badge('ok', 'Running') }}
{% elif c.status == 'restarting' %}{{ status_badge('crit', 'Restarting') }}
{% elif c.status == 'paused' %}{{ status_badge('warn', 'Paused') }}
{% elif exit_failed %}{{ status_badge('crit', c.status|capitalize) }}
{% else %}{{ status_badge('dim', c.status|capitalize) }}{% endif %}
{%- if c.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}
{%- elif c.health == 'starting' %} {{ status_badge('pending', 'Starting') }}
{%- elif c.health == 'healthy' %} {{ status_badge('ok', 'Healthy') }}{% endif %}
{%- if uptime %} · up {{ uptime }}{% endif %}
{%- if host %} · on <a href="/hosts/{{ host.id }}">{{ host.name }}</a>{% endif %}
{%- if c.service_name %} · task of <a href="/plugins/docker/service/{{ c.service_name }}">{{ c.service_name }}</a>
{%- elif c.compose_project %} · compose <span class="chip">{{ c.compose_project }}</span>{% endif %}
{%- if c.image %} · <span class="mono" style="overflow-wrap:anywhere;" title="{{ c.image }}">{{ c.image.split("@")[0] }}</span>{% endif %}
</p>
<p class="page-subtitle tone-dim" style="margin-top:0.2rem;">
{%- if label != c.name %}<span class="mono" style="overflow-wrap:anywhere;">{{ c.name }}</span> · {% endif -%}
{%- if c.scraped_at %}reported {{ c.scraped_at|ago }}{% endif -%}
</p>
</div>
<div class="page-actions"><a class="btn btn-sm btn-ghost" href="/plugins/docker/container/{{ host_id }}/{{ name }}/logs">Logs</a></div>
{# ── Header ──────────────────────────────────────────────────────────────── #}
<div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
<span class="dot {% if container.status == 'running' %}dot-up{% elif container.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<h1 class="page-title" style="margin-bottom:0;">{{ container.name }}</h1>
{% if container.health == 'healthy' %}<span style="font-size:0.72rem;color:var(--green);border:1px solid var(--green);border-radius:3px;padding:0.05rem 0.4rem;">healthy</span>
{% elif container.health == 'unhealthy' %}<span style="font-size:0.72rem;color:var(--red);border:1px solid var(--red);border-radius:3px;padding:0.05rem 0.4rem;">unhealthy</span>
{% elif container.health == 'starting' %}<span style="font-size:0.72rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0.05rem 0.4rem;">starting</span>{% endif %}
</div>
<div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
{{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %}
{% if host %} · on <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %}
</div>
{# ── Why it stopped, only while it has ───────────────────────────────────── #}
{% if c.status != 'running' and exit_failed %}
<section class="card problem-card" aria-labelledby="ctr-exit">
<h2 id="ctr-exit" class="section-title" style="margin-bottom:0.4rem;">Last exit</h2>
<p style="margin:0;">
Exited <span class="tone-crit">{{ c.exit_code }}</span>{% if c.oom_killed %}: killed for running out of memory{% endif %}
{%- if died_at %} · {{ died_at|ago }}{% endif %}
· <a href="/plugins/docker/container/{{ host_id }}/{{ name }}/logs">its logs →</a>
</p>
</section>
{% endif %}
{# ── Facts grid ──────────────────────────────────────────────────────────── #}
{% macro fact(label, value, colour="") %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">{{ label }}</div>
<div style="font-size:0.95rem;{% if colour %}color:{{ colour }};{% endif %}font-family:ui-monospace,monospace;">{{ value }}</div>
</div>
{% endmacro %}
{# ── What it is set up with, and what it moved since it started ─────────── #}
<div class="card">
<dl class="kv">
{% if not c.service_name %}
{#- Swarm never restarts a task's container: it replaces the task, so a
task's restart count is always 0; its service's Tasks table is the
count that means something. #}
<dt>Restarts</dt><dd>{% if c.restart_count %}<span class="tone-warn">{{ c.restart_count }}</span>{% else %}0{% endif %}
{%- if c.exit_code is not none and c.status == 'running' and c.restart_count %} <span class="tone-dim">· last exit {{ c.exit_code }}</span>{% endif %}</dd>
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
{{ fact("CPU", "%.1f%%" | format(container.cpu_pct) if container.cpu_pct is not none else "—") }}
{{ fact("Memory", "%.1f%%" | format(container.mem_pct) if container.mem_pct is not none else "—") }}
{{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }}
{{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—",
"var(--red)" if (container.exit_code is not none and container.exit_code != 0) else "") }}
{{ fact("Net in", net_rx) }}
{{ fact("Net out", net_tx) }}
{{ fact("Block read", blk_read) }}
{{ fact("Block write", blk_write) }}
</div>
{# ── Image / placement ───────────────────────────────────────────────────── #}
<div class="card" style="margin-bottom:1.5rem;">
<div style="display:grid;grid-template-columns:max-content 1fr;gap:0.4rem 1.25rem;font-size:0.86rem;">
<span style="color:var(--text-muted);">Image</span>
<span style="font-family:ui-monospace,monospace;word-break:break-all;">{{ container.image or "—" }}</span>
{% if container.compose_project %}
<span style="color:var(--text-muted);">Compose project</span><span>{{ container.compose_project }}</span>
{% endif %}
{% if container.service_name %}
<span style="color:var(--text-muted);">Swarm service</span><span>{{ container.service_name }}</span>
{% endif %}
{% if container.node_id %}
<span style="color:var(--text-muted);">Node</span><span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>
{% endif %}
<dt>Memory limit</dt><dd>{% if usage.mem_limit %}{{ usage.mem_limit|bytes }}{% else %}<span class="tone-dim">none set</span>{% endif %}</dd>
{% if ports %}
<dt>Ports</dt><dd class="mono">{% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %}</dd>
<span style="color:var(--text-muted);">Ports</span>
<span style="font-family:ui-monospace,monospace;">{% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %}</span>
{% endif %}
{% if c.node_id %}
<dt>Swarm node</dt><dd title="node id {{ c.node_id }}">{{ node_name or c.node_id[:12] }}</dd>
{% endif %}
<dt>Network</dt><dd>{{ net_rx }} in · {{ net_tx }} out <span class="tone-dim">since it started</span></dd>
<dt>Disk</dt><dd>{{ blk_read }} read · {{ blk_write }} written <span class="tone-dim">since it started</span></dd>
</dl>
</div>
</div>
{# ── Resource history: the one home for what it uses ────────────────────── #}
<section class="card" aria-labelledby="ctr-history">
{# ── Resource history (range-toggled) ────────────────────────────────────── #}
<div class="card" style="margin-bottom:1.5rem;">
<div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
<h2 id="ctr-history" class="section-title" style="margin:0;">Resource history</h2>
<h3 class="section-title" style="margin-bottom:0;">Resource history</h3>
{% include "_time_range.html" %}
</div>
<div id="container-history"
@@ -96,30 +80,31 @@
hx-trigger="load, rangeChange from:body"
hx-include="#time-range"
hx-swap="innerHTML">
<p class="field-hint">Loading…</p>
<div style="color:var(--text-muted);font-size:0.85rem;">Loading…</div>
</div>
</section>
</div>
{# ── Lifecycle, newest first ─────────────────────────────────────────────── #}
<section class="card-flush" aria-labelledby="ctr-lifecycle">
<h2 id="ctr-lifecycle" class="table-caption">Lifecycle <span class="tone-muted">newest first</span></h2>
{# ── Lifecycle timeline ──────────────────────────────────────────────────── #}
<div class="card">
<h3 class="section-title" style="margin-bottom:0.75rem;">Lifecycle</h3>
{% if events %}
<table class="table">
<thead><tr><th>Event</th><th>Detail</th><th class="num">When</th></tr></thead>
<tbody>
{% for e in events %}
<tr>
<td><span class="tone-{{ e.tone }}" aria-hidden="true">{{ e.glyph }}</span> {{ e.event|replace("_", " ") }}</td>
<td class="tone-muted" style="overflow-wrap:anywhere;">{{ e.detail or "" }}</td>
<td class="num">{{ e.at|ago }}</td>
</tr>
{% endfor %}
</tbody>
</table>
<div style="display:grid;gap:0.5rem;">
{% for e in events %}
<div style="display:flex;align-items:baseline;gap:0.6rem;font-size:0.85rem;">
<span style="color:{{ e.colour }};width:1rem;text-align:center;flex-shrink:0;">{{ e.glyph }}</span>
<span style="font-weight:500;width:6.5rem;flex-shrink:0;">{{ e.event }}</span>
<span style="color:var(--text-muted);flex:1;min-width:0;">{{ e.detail or "" }}</span>
<span style="color:var(--text-dim);font-size:0.78rem;white-space:nowrap;" title="{{ e.at }}">{{ e.at.strftime("%Y-%m-%d %H:%M") }}</span>
</div>
{% endfor %}
</div>
{% else %}
<p class="tone-muted" style="margin:0;padding:1rem;">No lifecycle events recorded yet. Starts, stops and health changes appear here as the agent reports them.</p>
<div style="color:var(--text-muted);font-size:0.85rem;">
No lifecycle events recorded yet. Start/stop/health changes appear here as the
agent reports them over time.
</div>
{% endif %}
</section>
</div>
{% endif %}
{% endblock %}
plugins/docker/templates/docker/container_logs.html
-51
View File
@@ -1,51 +0,0 @@
{# docker/container_logs.html — per-container log viewer (m79) #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs %}
{% block title %}Logs — {{ name }} — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([
("Docker", "/plugins/docker/"),
(name, "/plugins/docker/container/" ~ host_id ~ "/" ~ name),
("Logs", "")]) }}{% endblock %}
{% block content %}
<div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
<h1 class="page-title" style="margin-bottom:0;">{{ name }}</h1>
<span style="font-size:0.9rem;color:var(--text-muted);">logs</span>
</div>
<div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1rem;">
Recent lines (newest first) collected by the host agent{% if host %} on
<a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %} —
updated every few seconds.
<a href="/plugins/docker/container/{{ host_id }}/{{ name }}">← back to container</a>
</div>
{# ── Controls: stream filter + text search (drive the fragment via HTMX) ───── #}
<form id="log-controls" onsubmit="return false;"
style="display:flex;gap:0.6rem;align-items:center;flex-wrap:wrap;margin-bottom:0.6rem;">
<label style="font-size:0.8rem;color:var(--text-muted);display:flex;align-items:center;gap:0.35rem;">
Stream
<select name="stream" style="padding:0.25rem 0.4rem;">
<option value="all">all</option>
<option value="stdout">stdout</option>
<option value="stderr">stderr</option>
</select>
</label>
<input type="search" name="q" placeholder="Filter lines…" autocomplete="off"
aria-label="Filter log lines"
style="flex:1;min-width:180px;padding:0.3rem 0.5rem;">
</form>
<div class="card-flush">
<div id="log-lines"
hx-get="/plugins/docker/container/{{ host_id }}/{{ name }}/logs/lines"
hx-trigger="load, every 5s, change from:#log-controls, keyup changed delay:400ms from:#log-controls"
hx-include="#log-controls"
hx-swap="innerHTML"
role="log" aria-live="polite" tabindex="0"
style="max-height:70vh;overflow:auto;padding:0.5rem 0.75rem;
font-family:ui-monospace,monospace;font-size:0.8rem;line-height:1.5;">
<div style="color:var(--text-muted);">Loading…</div>
</div>
</div>
{% endblock %}
plugins/docker/templates/docker/disk.html
+63 -104
View File
@@ -1,124 +1,83 @@
{# docker/disk.html — what Docker keeps on each host's disk (#5304, #5305):
the hosts side by side with how much each grew, then each host's parts,
its cleanup actions, and its images folded under their repository. #}
{# docker/disk.html — image/disk usage: reclaimable space, per-image sizes #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header, stat_tile, status_badge %}
{% from "_macros.html" import crumbs %}
{% block title %}Disk — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Disk", "")]) }}{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Image & disk usage", "")]) }}{% endblock %}
{% block content %}
{% call page_header("Docker disk", "What Docker keeps on each host's disk, from docker system df, and how it has grown.") %}
{% include "_time_range.html" %}
{% endcall %}
<h1 class="page-title" style="margin-bottom:0.4rem;">Image &amp; disk usage</h1>
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
Reclaimable = space held by images no container references. Cleanup actions
(prune) arrive in a later release — these are read-only figures for now.
</p>
{% if host_groups %}
<section class="card-flush" aria-labelledby="disk-hosts">
<h2 id="disk-hosts" class="table-caption">By host <span class="tone-muted">most space first</span></h2>
<div hx-get="/plugins/docker/disk/hosts" hx-trigger="load, rangeChange from:body"
hx-include="#time-range" hx-swap="innerHTML">
<p class="field-hint" style="padding:1rem;margin:0;">Loading…</p>
</div>
</section>
{% for g in host_groups %}
{% set s = g.summary %}
<section id="host-{{ g.host_id }}" class="disk-host" aria-labelledby="disk-host-{{ g.host_id }}">
<h2 id="disk-host-{{ g.host_id }}" class="disk-host-head">
<a href="/hosts/{{ g.host_id }}">{{ g.host_name }}</a>
<span class="tone-muted">{{ g.total|bytes }}</span>
{% if g.as_of %}<span class="field-hint">as of {{ g.as_of|ago }}</span>{% endif %}
</h2>
<div style="margin-bottom:2rem;">
<div style="font-weight:600;font-size:0.95rem;margin-bottom:0.75rem;">{{ g.host_name }}</div>
<div class="stat-strip">
{{ stat_tile("Reclaimable", s.images_reclaimable|bytes, state="warn" if s.images_reclaimable else "ok", hint="held by images no container uses") }}
{{ stat_tile("Images", s.images_size|bytes, hint=s.images_active ~ " of " ~ s.images_total ~ " in use") }}
{{ stat_tile("Container files", s.containers_size|bytes, hint="written by " ~ s.containers_count ~ " containers on top of their images") }}
{{ stat_tile("Volumes", s.volumes_size|bytes, hint=s.volumes_count ~ " volumes; never pruned here") }}
{{ stat_tile("Build cache", s.build_cache_size|bytes, hint="left by docker build; safe to clear") }}
{{ stat_tile("Stopped", g.stopped, state="dim" if not g.stopped else none, hint="containers not running") }}
{# ── Summary stats ────────────────────────────────────────────────────── #}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1rem;">
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Reclaimable</div>
<span class="stat-val" style="color:{% if g.summary.images_reclaimable != '0 B' %}var(--orange){% else %}var(--green){% endif %};">{{ g.summary.images_reclaimable }}</span>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Images</div>
<span class="stat-val">{{ g.summary.images_size }}</span>
<div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.images_active }}/{{ g.summary.images_total }} in use</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Stopped</div>
<span class="stat-val" style="{% if g.stopped %}color:var(--text-muted){% endif %};">{{ g.stopped }}</span>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Writable layers</div>
<span class="stat-val">{{ g.summary.containers_size }}</span>
<div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.containers_count }} containers</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Volumes</div>
<span class="stat-val">{{ g.summary.volumes_size }}</span>
<div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.volumes_count }} volumes</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Build cache</div>
<span class="stat-val">{{ g.summary.build_cache_size }}</span>
</div>
</div>
{# ── Cleanup actions (admin only; audited Ansible prune run) ───────────── #}
{% if session.user_role == 'admin' %}
<div class="disk-actions">
{% if ansible_available and g.has_target %}
<form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune"
data-confirm="Remove all STOPPED containers on {{ g.host_name }}? This cannot be undone.">{{ csrf_field() }}
<input type="hidden" name="target" value="containers">
<button type="submit" class="btn btn-sm">Prune stopped containers</button>
</form>
<form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune"
data-confirm="Remove ALL unused images on {{ g.host_name }} (docker image prune -a)? Any image not used by a container is deleted.">{{ csrf_field() }}
<input type="hidden" name="target" value="images">
<button type="submit" class="btn btn-sm">Prune unused images</button>
</form>
<form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune"
data-confirm="Run a full system prune on {{ g.host_name }}? Removes stopped containers, unused networks, untagged images nothing uses, and the build cache.">{{ csrf_field() }}
<input type="hidden" name="target" value="system">
<button type="submit" class="btn btn-sm btn-danger">System prune…</button>
</form>
<span class="field-hint">Each runs an audited Ansible playbook on {{ g.host_name }}; reclaimed space shows on the next agent sample.</span>
{% elif not ansible_available %}
<span class="field-hint">Cleanup needs the Ansible runner (currently unavailable).</span>
{% else %}
<span class="field-hint">Link an <a href="/hosts/{{ g.host_id }}">Ansible target</a> to this host to enable prune actions.</span>
{% endif %}
</div>
{% endif %}
{# ── Images by repository, versions folded under it (#5305) ─────────── #}
<section class="card-flush" aria-label="Images on {{ g.host_name }}">
<h3 class="table-caption">Images by repository
<span class="tone-muted">{% if g.images_capped %}the {{ g.images_listed }} largest of {{ s.images_total }} images{% else %}{{ g.images_listed }} images{% endif %}</span></h3>
{% if g.repos %}
<table class="table image-repos">
<thead><tr>
<th>Repository</th>
<th class="num">Versions</th>
<th class="num" title="Each version's size less the layers it shares with other images, added up: what removing them all would free, roughly">Own size</th>
<th class="num">Used by</th>
<th class="num" title="Own size of the versions no container uses">Reclaimable</th>
</tr></thead>
{% for r in g.repos %}
<tbody class="image-repo">
{# ── Per-image table ──────────────────────────────────────────────────── #}
<div class="card-flush">
<table class="table">
<thead>
<tr><th>Image</th><th>Size</th><th>Shared</th><th>Containers</th></tr>
</thead>
<tbody>
{% for im in g.images %}
<tr>
<td class="cell-name">
<details>
<summary title="{{ r.repo or 'untagged images' }}">{% if r.repo %}<span class="mono">{{ r.repo }}</span>{% else %}<span class="tone-muted">untagged</span>{% endif %}</summary>
<table class="table image-versions">
<thead><tr><th>Tag</th><th>ID</th><th class="num">Size</th><th class="num" title="Layers in common with other images: stored once">Shared</th><th class="num">Used by</th></tr></thead>
<tbody>
{% for v in r.versions %}
<tr>
<td class="mono">{{ v.tag or "—" }}</td>
<td class="mono tone-muted">{{ (v.image_id or "")|replace("sha256:", "")|truncate(12, true, "") }}</td>
<td class="num">{{ v.size|bytes }}</td>
<td class="num tone-muted">{{ v.shared|bytes }}</td>
<td class="num">{% if v.containers %}{{ v.containers }} container{{ "s" if v.containers != 1 }}{% else %}<span class="tone-warn">nothing</span>{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
</details>
<td style="font-size:0.84rem;font-family:ui-monospace,monospace;max-width:340px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
{{ im.repo_tag }}
{% if im.reclaimable %}<span title="not referenced by any container" style="font-size:0.68rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0 0.3rem;margin-left:0.4rem;">reclaimable</span>{% endif %}
</td>
<td class="num">{{ r.versions|length }}{% if r.unused %} <span class="tone-dim">({{ r.unused }} unused)</span>{% endif %}</td>
<td class="num">{{ r.own|bytes }}</td>
<td class="num">{% if r.containers %}{{ r.containers }} container{{ "s" if r.containers != 1 }}{% else %}<span class="tone-warn">nothing</span>{% endif %}</td>
<td class="num">{% if r.reclaimable %}<span class="tone-warn">{{ r.reclaimable|bytes }}</span>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.86rem;">{{ im.size }}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.86rem;color:var(--text-muted);">{{ im.shared }}</td>
<td style="font-size:0.86rem;color:{% if im.containers %}var(--text){% else %}var(--text-muted){% endif %};">{{ im.containers }}</td>
</tr>
{% else %}
<tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No images reported.</td></tr>
{% endfor %}
</tbody>
{% endfor %}
</table>
{% else %}
<p class="tone-muted" style="margin:0;padding:1rem;">No images reported.</p>
{% endif %}
</section>
</section>
</div>
</div>
{% endfor %}
{% else %}
<div class="card empty-state">
No disk usage reported yet. The host agent reports what Docker keeps on disk,
from <code>docker system df</code>, on hosts running Docker.
<div class="card" style="text-align:center;padding:3rem;">
<div style="color:var(--text-muted);font-size:0.9rem;">
No disk usage reported yet. The host agent reports image/disk usage from
<code>docker system df</code> on hosts running Docker.
</div>
</div>
{% endif %}
{% endblock %}
plugins/docker/templates/docker/host_panel.html
+14 -68
View File
@@ -1,76 +1,22 @@
{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub:
problems first with what is wrong and since when, then the busiest running
containers; the rest of the running and the stopped fold away so the card
stays short (#5276). Tasks Swarm has replaced are left out (counted in the
header). #}
{% from "_macros.html" import status_badge, status_dot %}
{% from "docker/_usage.html" import cpu, mem %}
{% macro container_table(rows) %}
<table class="table">
<colgroup><col><col style="width:7rem"><col style="width:9rem"></colgroup>
<thead>
<tr>
<th>Container</th>
<th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
<th title="Used, and the limit when one is set">Memory</th>
</tr>
</thead>
<tbody>
{% for r in rows %}
{% set c = r.c %}
<tr>
<td>
<div style="display:flex;align-items:center;gap:0.5rem;min-width:0;">
{{ status_dot('crit' if r.problems else ('up' if c.status == 'running' else 'inactive'), c.status|capitalize) }}
<div style="min-width:0;">
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }} · {{ c.image }}" style="font-weight:500;color:inherit;text-decoration:none;overflow-wrap:anywhere;">{{ r.label }}</a>
{% if r.problems %}
{% set worst = r.problems[0] %}
<div style="font-size:0.75rem;margin-top:0.1rem;">
{{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
<span style="color:var(--text-muted);">
{%- for p in r.problems %} {{ says[p] }}{{ " and" if not loop.last }}{% endfor %}
{%- if c.exit_code %} · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}{% endif %}
{%- if r.since %} · since {{ r.since|ago }}{% endif -%}
</span>
</div>
{% else %}
<div style="font-size:0.72rem;color:var(--text-muted);">
{{ c.status }}{% if r.uptime %} · up {{ r.uptime }}{% endif %}
{%- if c.status != 'running' and c.exit_code %} · exit {{ c.exit_code }}{% endif %}
</div>
{% endif %}
</div>
</div>
</td>
<td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endmacro %}
{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub #}
<div class="card">
<div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;flex-wrap:wrap;">
<div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;">
<h3 class="section-title" style="margin-bottom:0;">Docker</h3>
<span style="font-size:0.78rem;color:var(--text-muted);">
{% if problems %}<span class="tone-crit">{{ problems }} with problems</span> · {% endif %}
{{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %}
{% if replaced %}<span title="Old swarm tasks Swarm has replaced with new ones: history, not containers that need anything"> · {{ replaced }} replaced task{{ "s" if replaced != 1 }} hidden</span>{% endif %}
<a href="/plugins/docker/" style="margin-left:0.6rem;color:var(--text-muted);">All →</a>
</span>
</div>
{{ container_table(rows) }}
{% if folded %}
<details style="margin-top:0.5rem;">
<summary style="cursor:pointer;font-size:0.82rem;color:var(--text-muted);">
{%- set more_running = folded|selectattr("c.status", "equalto", "running")|list|length -%}
{%- set more_stopped = folded|length - more_running -%}
{%- if more_running %}+{{ more_running }} more running{% endif -%}
{%- if more_running and more_stopped %} · {% endif -%}
{%- if more_stopped %}{{ more_stopped }} stopped{% endif -%}
</summary>
{{ container_table(folded) }}
</details>
{% endif %}
<div style="display:grid;gap:2px;">
{% for c in containers %}
<div style="display:flex;align-items:center;gap:0.5rem;font-size:0.85rem;padding:0.2rem 0;">
<span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;" title="{{ c.image }}">{{ c.name }}</a>
<span style="font-size:0.74rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
{% if c.cpu_pct is not none %}{{ "%.1f" | format(c.cpu_pct) }}%{% endif %}
{% if c.mem_pct is not none %} · {{ "%.1f" | format(c.mem_pct) }}%{% endif %}
</span>
</div>
{% endfor %}
</div>
</div>
plugins/docker/templates/docker/index.html
+15 -72
View File
@@ -1,81 +1,24 @@
{% extends "base.html" %}
{% from "_macros.html" import page_header %}
{# The Docker page. The filter bar sits outside the polled fragment, so a poll
never resets what is being typed; the fragment reads the bar on every poll,
and a change re-fetches it and puts the filters in the URL (#5299). #}
{% block title %}Docker — Steward{% endblock %}
{% block content %}
{% call page_header("Docker", "What needs attention first, then every service and container.") %}
{% if has_swarm %}<a class="btn btn-sm btn-ghost" href="/plugins/docker/swarm">Swarm</a>{% endif %}
{% if has_disk %}<a class="btn btn-sm btn-ghost" href="/plugins/docker/disk">Disk</a>{% endif %}
{% endcall %}
<form id="docker-filters" class="filter-bar" method="get" action="/plugins/docker/" role="search">
<input type="search" id="docker-q" name="q" value="{{ f.q }}" class="input"
placeholder="Search name, image or stack" aria-label="Search services and containers" autocomplete="off">
{% if choices.stacks %}
<select name="stack" aria-label="Stack">
<option value="">All stacks</option>
{% for s in choices.stacks %}<option value="{{ s }}"{% if f.stack == s %} selected{% endif %}>{{ s }}</option>{% endfor %}
<option value="{{ no_stack }}"{% if f.stack == no_stack %} selected{% endif %}>No stack</option>
</select>
{% endif %}
{% if choices.hosts|length > 1 %}
<select name="host" aria-label="Host or node">
<option value="">All hosts</option>
{% for h in choices.hosts %}<option value="{{ h }}"{% if f.host == h %} selected{% endif %}>{{ h }}</option>{% endfor %}
</select>
{% endif %}
<select name="state" aria-label="Status">
<option value="">Any status</option>
{% for key, label in states.items() %}<option value="{{ key }}"{% if f.state == key %} selected{% endif %}>{{ label }}</option>{% endfor %}
</select>
{% if f.sort != 'status' %}<input type="hidden" name="sort" value="{{ f.sort }}"><input type="hidden" name="dir" value="{{ f.dir }}">{% endif %}
<noscript><button type="submit" class="btn btn-sm">Filter</button></noscript>
</form>
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1.5rem;gap:1rem;flex-wrap:wrap;">
<div style="display:flex;align-items:baseline;gap:1rem;">
<h1 class="page-title" style="margin-bottom:0;">Docker</h1>
{% if has_swarm %}
<a href="/plugins/docker/swarm" style="font-size:0.85rem;color:var(--accent);text-decoration:none;">Swarm →</a>
{% endif %}
{% if has_disk %}
<a href="/plugins/docker/disk" style="font-size:0.85rem;color:var(--accent);text-decoration:none;">Disk →</a>
{% endif %}
</div>
{% include "_time_range.html" %}
</div>
<div id="docker-rows"
hx-get="/plugins/docker/rows"
hx-include="#docker-filters"
hx-trigger="load, every {{ poll_interval }}s, change from:#docker-filters, input changed delay:250ms from:#docker-q, search from:#docker-q"
hx-trigger="load, rangeChange from:body"
hx-include="#time-range"
hx-swap="innerHTML">
<div class="card"><p class="field-hint">Loading…</p></div>
<div style="color:var(--text-muted);font-size:0.9rem;padding:2rem;">Loading...</div>
</div>
{% if disk %}
{# What Docker keeps on each host's disk, and its growth over a week (#5306). #}
<p class="docker-disk-line">
<a href="/plugins/docker/disk" class="section-title">Disk</a>
{% for d in disk %}
<span class="docker-disk-host"><a href="/plugins/docker/disk#host-{{ d.host_id }}">{{ d.host_name }}</a>
{{ d.total|bytes }}
{%- if d.growth and d.growth.change %} <span class="tone-{{ 'warn' if d.growth.pct is not none and d.growth.pct >= 10 else 'muted' }}"
title="Since {{ d.growth.since.strftime('%Y-%m-%d %H:%M UTC') }}">{{ "+" if d.growth.change > 0 else "−" }}{{ (d.growth.change|abs)|bytes }} in 7d</span>{% endif -%}
{%- if d.reclaimable %} <span class="tone-dim">· {{ d.reclaimable|bytes }} reclaimable</span>{% endif %}</span>
{% endfor %}
</p>
{% endif %}
<script>
(function () {
var rows = document.getElementById('docker-rows');
var form = document.getElementById('docker-filters');
// The filters live in the URL too, so a filtered view can be bookmarked and
// Back returns to it.
function keepInUrl() {
var q = new URLSearchParams(new FormData(form));
Array.from(q.keys()).forEach(function (k) { if (!q.get(k)) q.delete(k); });
var s = q.toString();
history.replaceState(null, '', location.pathname + (s ? '?' + s : '') + location.hash);
}
form.addEventListener('change', keepInUrl);
form.addEventListener('input', keepInUrl);
// A link to a section (the dashboard's "failed (24h)" → #failures) lands
// before the rows exist; scroll to it once, when they first arrive.
if (!location.hash || !rows) return;
rows.addEventListener('htmx:afterSwap', function once() {
rows.removeEventListener('htmx:afterSwap', once);
var target = document.getElementById(location.hash.slice(1));
if (target) target.scrollIntoView({ block: 'start' });
});
})();
</script>
{% endblock %}
plugins/docker/templates/docker/rows.html
+170 -224
View File
@@ -1,228 +1,174 @@
{# docker/rows.html — HTMX fragment for the Docker page (#5298–#5301):
a verdict line; what needs
attention (service.needs_attention, the ranking the MCP tools use); hosts
whose Docker is not answering; what failed in 24h, each with its logs; then
the swarm services and the standalone containers as two tables that share
the filter bar's search, filters and sort (overview.py). #}
{% from "_macros.html" import status_badge, status_dot, sort_th %}
{% from "docker/_usage.html" import cpu, mem, cores %}
{% macro th(key, label, cls=none, title=none) -%}
{{ sort_th(label, sort_url(key), active=f.sort == key, desc=f.dir == 'desc', cls=cls, title=title) }}
{%- endmacro %}
{% macro up(row) -%}
{%- if row.started_at %}<span title="Started {{ row.started_at.strftime('%Y-%m-%d %H:%M UTC') }}">{{ row.started_at|ago|striptags|replace(" ago", "") }}</span>
{%- else %}<span class="tone-dim">—</span>{% endif -%}
{%- endmacro %}
{# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #}
{# ── Verdict ──────────────────────────────────────────────────────────────── #}
{% set problems = attention|length + failing_services|length %}
<p class="page-verdict">
{% if problems %}<strong class="tone-crit">{{ problems }} need{{ "s" if problems == 1 }} attention</strong>
{% else %}<strong class="tone-ok">Nothing needs attention</strong>{% endif %}
{%- if services_total %} · <span class="tone-{{ 'ok' if services_healthy == services_total else 'warn' }}">{{ services_healthy }}/{{ services_total }}</span> services healthy{% endif %}
{%- if containers_total %} · {{ running }} container{{ "s" if running != 1 }} running{% if stopped %}, {{ stopped }} stopped{% endif %}{% if services_total %} outside services{% endif %}{% endif %}
{%- if docker_hosts %} · {{ docker_hosts }} Docker host{{ "s" if docker_hosts != 1 }}{% endif %}
{%- if stale %} · <span class="tone-warn" title="No recent report from their agent: what is shown may no longer be true">{{ stale }} stale</span>{% endif %}
</p>
{# ── Needs attention ──────────────────────────────────────────────────────── #}
{% if attention or failing_services %}
<section class="card problem-card" aria-labelledby="docker-problems">
<h2 id="docker-problems" class="section-title section-count">Needs attention <span class="tone-crit">{{ problems }}</span>
{%- if unhealthy %}<span class="tone-muted"> · {{ unhealthy }} unhealthy</span>{% endif %}
{%- if restarting %}<span class="tone-muted"> · {{ restarting }} restarting</span>{% endif %}</h2>
<ul class="problem-list">
{# A swarm service in trouble: Swarm replaces its failed tasks, so the
service, not any one container, is what is failing. #}
{% for s in failing_services %}
{% set e = s.last_error %}
<li>
{% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
{% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}
{% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
<a href="/plugins/docker/service/{{ s.name }}">{{ s.name }}</a>
<span class="problem-detail">
{{- s.running }} of {{ s.desired }} running
{%- if s.crash_looping %} · {{ s.failures_1h }} tasks failed in the last hour{% endif %}
{%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.exit_code %} (exit {{ e.exit_code }}){% endif %}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
{%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
</span>
<a class="field-hint" href="/plugins/docker/service/{{ s.name }}{% if e and e.task_id %}?task={{ e.task_id }}{% endif %}#logs" style="margin-left:0.4rem;">Logs →</a>
</li>
{% endfor %}
{% for r in attention %}
{% set c = r.container %}
{% set worst = r.problems[0] %}
<li>
{{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }}">{{ r.label }}</a>
<span class="problem-detail">
{%- for p in r.problems %}{{ says[p] }}{{ " and " if not loop.last }}{% endfor %}
{%- if c.exit_code and worst in ('exited_nonzero', 'crash_looping', 'restarting') %} · exit {{ c.exit_code }}{% endif %}
{%- if r.since %} · since {{ r.since|ago }}{% endif %}
{%- if r.stale %} · {{ status_badge('stale') }} last report {{ c.scraped_at|ago if c.scraped_at else "never" }}{% endif -%}
</span>
<a class="field-hint" href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}/logs" style="margin-left:0.4rem;">Logs →</a>
</li>
{% endfor %}
</ul>
</section>
{% endif %}
{# ── Hosts whose Docker is not answering (agent ≥ 1.9.0) ──────────────────── #}
{% if docker_down %}
<section class="card problem-card" aria-labelledby="docker-unreachable">
<h2 id="docker-unreachable" class="section-title" style="margin-bottom:0.6rem;">Docker not answering</h2>
<ul class="problem-list">
{% for d in docker_down %}
<li>
{{ status_badge('crit', 'Error') if d.status == 'error' else status_badge('stale', 'No Docker') }}
<a href="/hosts/{{ d.host_id }}">{{ d.host_name }}</a>
<span class="problem-detail">{{ d.detail or "no reason given" }} · since {{ d.since|ago }} · its containers below are what Docker last reported</span>
</li>
{% endfor %}
</ul>
</section>
{% endif %}
{# ── What failed in the last 24h: what the dashboard's "failed (24h)" counts.
A plain container's death, or a swarm service's failed tasks; a container
someone stopped (exit 137/143) is not a failure. Each links to the logs of
what died: a service's to its failed task's lines (#5300). ───────────── #}
{% if failures %}
<section id="failures" class="card" aria-labelledby="docker-failures">
<h2 id="docker-failures" class="section-title section-count">Failed in the last 24h <span class="tone-warn">{{ failures|length }}</span></h2>
<ul class="problem-list">
{% for x in failures %}
<li>
{{ status_badge('warn', 'Service' if x.kind == 'service' else 'Container') }}
{% if x.kind == 'service' %}<a href="/plugins/docker/service/{{ x.name }}">{{ x.name }}</a>
{% else %}<a href="/plugins/docker/container/{{ x.host_id }}/{{ x.name }}">{{ x.name }}</a>{% endif %}
<span class="problem-detail">
{{- x.count }} {{ "failure" if x.count == 1 else "failures" }}
{%- if x.detail %} · last: {{ x.detail }}{% endif %}
{%- if x.host %} · {{ x.host }}{% endif %}
{%- if x.at %} · {{ x.at|ago }}{% endif -%}
</span>
{% if x.kind == 'service' %}
<a class="field-hint" href="/plugins/docker/service/{{ x.name }}{% if x.task_id %}?task={{ x.task_id }}{% endif %}#logs" style="margin-left:0.4rem;">Logs{% if x.task_id %} of that task{% endif %} →</a>
{% else %}
<a class="field-hint" href="/plugins/docker/container/{{ x.host_id }}/{{ x.name }}/logs" style="margin-left:0.4rem;">Logs →</a>
{% endif %}
</li>
{% endfor %}
</ul>
</section>
{% endif %}
{# ── Swarm services: one row each, problems first unless a header picked
another order; CPU and memory add up the running replicas (#5299). ───── #}
{% if services %}
<section class="card-flush" aria-labelledby="docker-services">
<h2 id="docker-services" class="table-caption">Swarm services <span class="tone-muted">{{ services|length }}{% if services|length != services_total %} of {{ services_total }}{% endif %}</span>
<a href="/plugins/docker/swarm" class="field-hint">By node →</a></h2>
<table class="table">
<thead>
<tr>
{{ th("name", "Service") }}
{{ th("status", "Status", title="Problems first: a crash loop, then down, then short of replicas, then failed tasks in 24h") }}
{{ th("stack", "Stack", title="The part of the service's name before the first _: the stack it was deployed in") }}
<th title="The nodes running its replicas">Nodes</th>
{{ th("cpu", "CPU", "num", "Across its running replicas; Docker counts CPU per core") }}
{{ th("mem", "Memory", "num", "Across its running replicas") }}
{{ th("up", "Up", "num", "Since its newest replica started") }}
<th class="num" title="Tasks that failed in the last 24h; Swarm replaced them">Failed 24h</th>
</tr>
</thead>
<tbody>
{% for r in services %}
<tr>
<td class="cell-name"><a href="/plugins/docker/service/{{ r.name }}" title="{{ r.name }}{% if r.image %} · {{ r.image }}{% endif %}">{{ r.name }}</a></td>
<td>
{{ status_dot('up' if r.state == 'healthy' else ('warn' if r.state == 'degraded' else 'down')) }}
<span class="tone-{{ 'ok' if r.state == 'healthy' else ('warn' if r.state == 'degraded' else 'crit') }}" style="font-variant-numeric:tabular-nums;">{{ r.running }}/{{ r.desired }}</span>
{%- if r.crash_looping %} {{ status_badge('crit', 'Crash-looping') }}{% elif r.unhealthy %} {{ status_badge('crit', 'Unhealthy') }}{% endif %}
{%- if r.mode == 'global' %} <span class="tone-dim">global</span>{% endif %}
</td>
<td>{% if r.stack %}<a class="chip" href="{{ page_url(stack=r.stack) }}" title="Show the {{ r.stack }} stack">{{ r.stack }}</a>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="tone-muted">
{{- r.nodes|join(", ") or ("" if r.ghost_nodes else "—") -}}
{%- if r.ghost_nodes %}{{ ", " if r.nodes }}<span title="Running on a node with no Steward agent: no figures for it">{{ r.ghost_nodes|join(", ") }} (no agent)</span>{% endif -%}
</td>
<td class="num">{% if r.cpu_cores is not none %}{{ cores(r.cpu_cores) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{% if r.mem_bytes is not none %}{{ r.mem_bytes|bytes }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{{ up(r) }}</td>
<td class="num">
{%- if r.failures_24h %}<a class="tone-warn" href="/plugins/docker/service/{{ r.name }}#tasks"
title="{% if r.last_error %}Last: {{ r.last_error.error or r.last_error.message or r.last_error.state }}{% if r.last_error.exit_code %} (exit {{ r.last_error.exit_code }}){% endif %}{% endif %}">{{ r.failures_24h }}</a>
{%- else %}<span class="tone-dim">—</span>{% endif -%}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</section>
{% endif %}
{# ── Containers outside any swarm service, every host in one table (#5301);
a compose project is its stack. ─────────────────────────────────────── #}
{% if containers %}
<section class="card-flush" aria-labelledby="docker-containers">
<h2 id="docker-containers" class="table-caption">{{ "Other containers" if services_total else "Containers" }} <span class="tone-muted">{{ containers|length }}{% if containers|length != containers_total %} of {{ containers_total }}{% endif %}</span></h2>
<table class="table">
<thead>
<tr>
{{ th("name", "Container") }}
{{ th("status", "Status", title="Problems first: unhealthy or restarting, then died with a failing exit, then stale; then running, then stopped") }}
<th>Host</th>
{{ th("stack", "Stack", title="Its compose project") }}
<th>Image</th>
{{ th("cpu", "CPU", "num", "Docker counts CPU per core: 1 core = one core fully busy") }}
{{ th("mem", "Memory", "num", "Used, and the limit when one is set") }}
{{ th("up", "Up", "num", "Since it started") }}
</tr>
</thead>
<tbody>
{% for r in containers %}
<tr>
<td class="cell-name">
<a href="/plugins/docker/container/{{ r.host_id }}/{{ r.container_name }}" title="{{ r.container_name }}">{{ r.name }}</a>
{%- if r.ports %}<div class="cell-sub mono">{% for p in r.ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{{ " " if not loop.last }}{% endfor %}</div>{% endif %}
</td>
<td>
{{ status_dot('stale' if r.stale else ('up' if r.running else ('pending' if r.status == 'paused' else 'down'))) }}
{%- if r.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}
{%- elif r.health == 'starting' %} {{ status_badge('pending', 'Starting') }}
{%- else %} <span class="tone-muted">{{ r.status }}</span>{% endif %}
{%- if not r.running and r.exit_code is not none and r.exit_code != 0 %}
<span class="tone-{{ 'crit' if r.exit_failed else 'dim' }}" title="{{ 'last exit code' if r.exit_failed else 'stopped by a signal; not a failure' }}">exit {{ r.exit_code }}{% if r.oom_killed %} (OOM){% endif %}</span>
{%- endif %}
{%- if r.restart_count %} <span class="tone-warn" title="Restarted {{ r.restart_count }} times">⟳{{ r.restart_count }}</span>{% endif %}
{%- if r.stale %} {{ status_badge('stale', title="Last report " ~ (r.scraped_at.strftime('%Y-%m-%d %H:%M UTC') if r.scraped_at else 'never') ~ "; what is shown may no longer be true") }}{% endif %}
</td>
<td class="tone-muted">{{ r.host }}</td>
<td>{% if r.stack %}<a class="chip" href="{{ page_url(stack=r.stack) }}" title="Show the {{ r.stack }} stack">{{ r.stack }}</a>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="cell-image" title="{{ r.image }}">{{ r.image }}</td>
<td class="num">{% if r.running %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{% if r.running %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{{ up(r) }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</section>
{% endif %}
{% if not services and not containers %}
<div class="card">
{% if services_total or containers_total %}
<p class="tone-muted" style="margin:0;">Nothing matches these filters. <a href="/plugins/docker/">Show everything</a></p>
{% else %}
<p class="tone-muted" style="margin:0;">
No containers reported yet. Containers are collected by the Steward host agent —
deploy the agent to a host running Docker (Hosts → the host → agent panel) and
its containers will appear here.
</p>
{# ── Summary strip ─────────────────────────────────────────────────────────── #}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(140px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Running</div>
<span class="stat-val" style="color:var(--green);">{{ running }}</span>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Stopped</div>
<span class="stat-val" style="{% if stopped %}color:var(--text-muted){% endif %};">{{ stopped }}</span>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Total</div>
<span class="stat-val">{{ total }}</span>
</div>
{% if swarm_services %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Services</div>
<span class="stat-val">{{ swarm_services | length }}</span>
</div>
{% endif %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Hosts</div>
<span class="stat-val">{{ host_groups | length }}</span>
</div>
</div>
{# ── Swarm services: collapsed per-service, each replica labelled with its host.
Ghost replicas (tasks on nodes with no Steward agent) come from the managers'
placement data so the picture is cluster-complete. ──────────────────────── #}
{% if swarm_services %}
<div class="card-flush" style="margin-bottom:1.5rem;">
<div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
<span style="font-weight:600;font-size:0.95rem;">Swarm services</span>
<a href="/plugins/docker/swarm" style="font-size:0.78rem;color:var(--text-muted);">Topology →</a>
</div>
<div style="padding:0.5rem 0.75rem;display:grid;gap:0.85rem;">
{% for s in swarm_services %}
<div>
<div style="display:flex;align-items:center;gap:0.55rem;flex-wrap:wrap;margin-bottom:0.35rem;">
<span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
<span style="font-weight:600;font-size:0.9rem;">{{ s.name }}</span>
<span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
<span style="font-size:0.78rem;color:{% if s.state == 'healthy' %}var(--green){% elif s.state == 'degraded' %}var(--orange){% else %}var(--red){% endif %};font-variant-numeric:tabular-nums;">
{{ s.running }}/{{ s.desired }} running
</span>
</div>
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(210px,1fr));gap:0.4rem;padding-left:1.1rem;">
{% for r in s.replicas %}
{% if r.ghost %}
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:5px;padding:0.3rem 0.55rem;" title="Running on a node with no Steward agent — detail unavailable">
<span class="dot dot-warn" style="opacity:0.6;"></span>
<span style="font-weight:500;">{{ r.host }}</span>
<span style="color:var(--text-dim);margin-left:auto;">{{ r.count }} · no agent</span>
</div>
{% else %}
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px solid var(--border);border-radius:5px;padding:0.3rem 0.55rem;">
<span class="dot {% if r.status == 'running' %}dot-up{% elif r.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" style="font-weight:500;color:inherit;text-decoration:none;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" title="{{ r.name }}">{{ r.host }}</a>
{% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;">{{ "%.0f" | format(r.cpu_pct) }}%</span>{% endif %}
</div>
{% endif %}
{% endfor %}
</div>
</div>
{% endfor %}
</div>
</div>
{% endif %}
{# ── Container tables, one per host ───────────────────────────────────────── #}
{% if host_groups %}
{% for g in host_groups %}
<div class="card-flush" style="margin-bottom:1.5rem;">
<div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
{% if g.host %}
<a href="/hosts/{{ g.host.id }}" style="font-weight:600;font-size:0.95rem;color:inherit;text-decoration:none;">{{ g.host.name }}</a>
{% else %}
<span style="font-weight:600;font-size:0.95rem;">{{ g.host_name }}</span>
{% endif %}
<span style="font-size:0.78rem;color:var(--text-muted);">
{{ g.running }} running{% if g.stopped %} · {{ g.stopped }} stopped{% endif %}
</span>
</div>
<table class="table">
<thead>
<tr>
<th>Container</th>
<th>Image</th>
<th>Ports</th>
<th>CPU %</th>
<th style="min-width:100px;">CPU history</th>
<th>Mem %</th>
<th style="min-width:100px;">Mem history</th>
</tr>
</thead>
<tbody>
{% for sub in g.subgroups %}
{% if g.grouped and sub.label %}
<tr><td colspan="7" style="padding:0.4rem 0.75rem 0.2rem;font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;font-weight:600;">{{ sub.label }}</td></tr>
{% endif %}
{% for item in sub.containers %}
{% set c = item.container %}
<tr>
<td>
<div style="display:flex;align-items:center;gap:0.5rem;">
<span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<div>
<div style="font-weight:500;font-size:0.9rem;">
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="color:inherit;text-decoration:none;border-bottom:1px dotted var(--border-mid);">{{ c.name }}</a>
{% if c.health == 'healthy' %}<span title="healthy" style="color:var(--green);font-size:0.7rem;">●</span>
{% elif c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);font-size:0.7rem;">●</span>
{% elif c.health == 'starting' %}<span title="health: starting" style="color:var(--orange);font-size:0.7rem;">◐</span>{% endif %}
</div>
<div style="font-size:0.73rem;color:var(--text-muted);">
{{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
{% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
· <span style="color:var(--red);">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
{% endif %}
{% if c.restart_count %} · <span title="restart count" style="color:var(--orange);">⟳{{ c.restart_count }}</span>{% endif %}
</div>
{% if c.service_name or c.compose_project %}
<div style="font-size:0.68rem;color:var(--text-dim);margin-top:0.1rem;">
{{ c.service_name or c.compose_project }}
</div>
{% endif %}
</div>
</div>
</td>
<td style="font-size:0.82rem;color:var(--text-muted);max-width:200px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
{{ c.image }}
</td>
<td style="font-size:0.78rem;font-family:ui-monospace,monospace;">
{% for p in item.ports %}
<div style="color:var(--text-muted);">{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}</div>
{% endfor %}
</td>
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
{% if c.cpu_pct is not none %}
<span style="color:{% if c.cpu_pct > 80 %}var(--red){% elif c.cpu_pct > 50 %}var(--orange){% else %}var(--text){% endif %};">
{{ "%.1f" | format(c.cpu_pct) }}%
</span>
{% else %}
<span style="color:var(--text-dim);">—</span>
{% endif %}
</td>
<td>{{ item.sparkline_cpu | safe }}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
{% if c.mem_pct is not none %}
<span style="color:{% if c.mem_pct > 90 %}var(--red){% elif c.mem_pct > 70 %}var(--orange){% else %}var(--text){% endif %};">
{{ "%.1f" | format(c.mem_pct) }}%
</span>
{% else %}
<span style="color:var(--text-dim);">—</span>
{% endif %}
</td>
<td>{{ item.sparkline_mem | safe }}</td>
</tr>
{% endfor %}
{% endfor %}
</tbody>
</table>
</div>
{% endfor %}
{% elif not swarm_services %}
<div class="card" style="text-align:center;padding:3rem;">
<div style="color:var(--text-muted);font-size:0.9rem;">
No containers reported yet. Containers are collected by the Steward host agent —
deploy the agent to a host running Docker (Hosts → the host → agent panel) and
its containers will appear here under that host.
</div>
</div>
{% endif %}
plugins/docker/templates/docker/service.html
-160
View File
@@ -1,160 +0,0 @@
{# docker/service.html — one swarm service (#5302, #5310): a header that says
whether it is well and what it runs; its last failure only while it is in
trouble; its CPU and memory over time; one table of its tasks, the running
ones first with what they use; and the logs of every task it has run.
A failure's "Logs →" lands on #logs with ?task= picked. #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, status_badge %}
{% from "docker/_usage.html" import cpu, mem %}
{% block title %}{{ name }} — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
{% block content %}
{% if service is none %}
<div class="card empty-state">
<strong>{{ "No longer running" if task_choices else "Service not found" }}</strong>
{% if task_choices %}
<span>No swarm service named <code>{{ name }}</code> is reported now; it may have been removed.
The logs Steward still keeps from its tasks are below.</span>
{% else %}
<span>No swarm service named <code>{{ name }}</code> is reported, and Steward holds no logs or
events for one. It may have been removed. <a href="/plugins/docker/">Back to Docker</a></span>
{% endif %}
</div>
{% else %}
{# ── Header: the verdict, then what it is (#5310). Each fact has one home:
what it uses is in Resource history, its tasks and failures in Tasks. #}
{% set troubled = row.crash_looping or row.state != 'healthy' %}
{% set tone = 'ok' if row.state == 'healthy' else ('warn' if row.state == 'degraded' else 'crit') %}
{% set image = (svc.image if svc and svc.image else row.image) %}
{% set e = row.last_error %}
<div class="page-header">
<div style="min-width:0;">
<h1 class="page-title" style="overflow-wrap:anywhere;">{{ name }}</h1>
<p class="page-subtitle">
{% if row.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
{% elif row.state == 'down' %}{{ status_badge('crit', 'Down') }}
{% elif row.state == 'degraded' %}{{ status_badge('warn', 'Degraded') }}
{% else %}{{ status_badge('ok', 'Healthy') }}{% endif %}
<span class="tone-{{ tone }}">{{ row.running }} of {{ row.desired }} running</span>
{%- if troubled and row.trouble_since %} · <span class="tone-{{ tone }}">in trouble since {{ row.trouble_since|ago }}</span>{% endif %}
· {{ row.mode }}
{%- if stack %} · stack <a class="chip" href="/plugins/docker/?stack={{ stack }}">{{ stack }}</a>{% endif %}
{%- if image %} · <span class="mono" style="overflow-wrap:anywhere;" title="{{ image }}">{{ image.split("@")[0] }}</span>{% endif %}
</p>
{% if (not troubled and e) or svc %}
<p class="page-subtitle tone-dim" style="margin-top:0.2rem;">
{%- if not troubled and e %}Last failed {% if e.age_s is not none %}{{ e.age_s|duration }} ago{% endif %}
{%- if e.exit_code %} (exit {{ e.exit_code }}){% endif %}
{%- if e.task_id %} · <a href="?task={{ e.task_id }}#logs">its logs</a>{% endif %} · {% endif -%}
{%- if svc %}reported {{ svc.updated_at|ago }} by the manager's agent{% endif -%}
</p>
{% endif %}
</div>
<div class="page-actions"><a class="btn btn-sm btn-ghost" href="#logs">Logs</a></div>
</div>
{# ── What is wrong, only while it is (#5310) ──────────────────────────────── #}
{% if troubled and e %}
<section class="card problem-card" aria-labelledby="svc-last-error">
<h2 id="svc-last-error" class="section-title" style="margin-bottom:0.4rem;">Last failed task</h2>
<p style="margin:0;">
{{ e.error or e.message or e.state }}{% if e.exit_code %} <span class="tone-crit">(exit {{ e.exit_code }})</span>{% endif %}
{%- if e.node %} · on {{ e.node }}{% endif %}{% if e.age_s is not none %} · {{ e.age_s|duration }} ago{% endif %}
{%- if e.task_id %} · <a href="?task={{ e.task_id }}#logs">its logs →</a>{% endif %}
</p>
</section>
{% endif %}
{# ── Resource history, summed across replicas ────────────────────────────── #}
<section class="card" aria-labelledby="svc-history">
<div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
<h2 id="svc-history" class="section-title" style="margin:0;">Resource history <span class="tone-dim">across running replicas</span></h2>
{% include "_time_range.html" %}
</div>
<div hx-get="/plugins/docker/service/{{ name }}/history"
hx-trigger="load, rangeChange from:body"
hx-include="#time-range"
hx-swap="innerHTML">
<p class="field-hint">Loading…</p>
</div>
</section>
{# ── Tasks: the running ones with what they use, then Swarm's record ─────── #}
<section id="tasks" class="card-flush" aria-labelledby="svc-tasks">
<h2 id="svc-tasks" class="table-caption">Tasks
<span class="tone-muted">running first, then Swarm's most recent</span>
{%- if row.failures_24h %} <span class="tone-warn">· {{ row.failures_24h }} failed in 24h</span>{% endif %}</h2>
{% if tasks %}
<table class="table">
<thead><tr>
<th title="The replica slot this task filled">Slot</th><th>Task</th><th>Node</th><th>State</th>
<th class="num" title="Docker counts CPU per core">CPU</th><th class="num">Memory</th>
<th class="num">Exit</th><th>Why</th><th class="num" title="When it started, for a running task; else when it reached its state">When</th>
</tr></thead>
<tbody>
{% for t in tasks %}
<tr>
<td>{{ t.slot or "—" }}</td>
<td class="mono" title="{{ t.container or t.task_id }}">
{%- if t.task_id %}<a href="?task={{ t.task_id }}#logs" title="Its logs">{{ t.task_id[:10] }}</a>
{%- if t.container and t.host_id %} <a class="tone-dim" href="/plugins/docker/container/{{ t.host_id }}/{{ t.container }}" title="The container's own page">↗</a>{% endif %}
{%- elif t.container and t.host_id %}<a href="/plugins/docker/container/{{ t.host_id }}/{{ t.container }}">{{ t.container|truncate(24, true, "…") }}</a>
{%- else %}<span class="tone-dim">—</span>{% endif -%}
</td>
<td>{% if t.host_id %}<a href="/hosts/{{ t.host_id }}">{{ t.node }}</a>{% else %}{{ t.node }}{% endif %}</td>
<td>
{%- if t.failed %}{{ status_badge('crit', t.state|capitalize) }}
{%- elif t.current %}{{ status_badge('ok', 'Running') }}
{%- else %}<span class="tone-muted">{{ t.state }}</span>{% endif %}
{%- if t.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}{% elif t.health == 'starting' %} {{ status_badge('pending', 'Starting') }}{% endif %}
{%- if t.desired and t.desired != t.state and not t.current %} <span class="tone-dim" title="What Swarm wants of it">→ {{ t.desired }}</span>{% endif -%}
</td>
<td class="num">{% if t.usage %}{{ cpu(t.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{% if t.usage %}{{ mem(t.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{% if t.exit_code %}<span class="tone-{{ 'crit' if t.failed else 'dim' }}">{{ t.exit_code }}</span>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td style="overflow-wrap:anywhere;">{{ t.why or "" }}</td>
<td class="num">{% if t.at %}{{ t.at|ago }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p class="tone-muted" style="margin:0;padding:1rem;">No task is running, and Swarm's record of its tasks has not been reported yet (the manager's agent reports it from version 1.8.0).</p>
{% endif %}
</section>
{% endif %}
{# ── Logs across every task, newest first ─────────────────────────────────
Shown for a removed service too, while its lines are still kept. #}
{% if service is not none or task_choices %}
<section id="logs" class="card-flush" aria-labelledby="svc-logs">
<h2 id="svc-logs" class="table-caption">Logs <span class="tone-muted">every task, newest first</span></h2>
<form id="svc-log-controls" class="filter-bar" onsubmit="return false;" style="padding:0.75rem 1rem 0;">
<input type="search" name="q" class="input" placeholder="Filter lines…" autocomplete="off" aria-label="Filter log lines">
{% if task_choices|length > 1 %}
<select name="task" aria-label="Task">
<option value="">All tasks</option>
{% for tid, label in task_choices %}<option value="{{ tid }}"{% if tid == focus_task %} selected{% endif %}>{{ label }}</option>{% endfor %}
</select>
{% elif focus_task %}<input type="hidden" name="task" value="{{ focus_task }}">{% endif %}
<select name="stream" aria-label="Stream">
<option value="all">stdout and stderr</option>
<option value="stdout">stdout</option>
<option value="stderr">stderr</option>
</select>
</form>
<div id="svc-log-lines"
hx-get="/plugins/docker/service/{{ name }}/logs/lines"
hx-trigger="load, every 5s, change from:#svc-log-controls, keyup changed delay:400ms from:#svc-log-controls"
hx-include="#svc-log-controls"
hx-swap="innerHTML"
role="log" aria-live="polite" tabindex="0"
style="max-height:60vh;overflow:auto;padding:0.5rem 1rem 0.75rem;font-family:var(--fs-font-mono);font-size:0.8rem;line-height:1.5;">
<p class="field-hint">Loading…</p>
</div>
</section>
{% endif %}
{% endblock %}
plugins/docker/templates/docker/swarm.html
+73 -89
View File
@@ -1,103 +1,87 @@
{# docker/swarm.html — the Swarm laid out by node (#5303). A verdict, the
services short of replicas, then one card per node: a node that is down or
not taking work first, then managers. Each card says what its containers use
against what the machine has, and lists the services placed on it. #}
{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, status_badge, status_dot %}
{% from "docker/_usage.html" import cores as fmt_cores %}
{% from "_macros.html" import crumbs %}
{% block title %}Swarm — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Swarm", "")]) }}{% endblock %}
{% block content %}
<h1 class="page-title">Swarm</h1>
<h1 class="page-title" style="margin-bottom:1.5rem;">Swarm</h1>
{% if swarms %}
{% for g in swarms %}
{% set down_nodes = g.nodes|rejectattr("ok")|list %}
<section class="swarm-group" aria-label="Swarm reported by {{ g.managers|join(', ') }}">
<p class="page-verdict">
{% if down_nodes or g.troubled %}<strong class="tone-crit">{{ down_nodes|length + g.troubled|length }} need{{ "s" if (down_nodes|length + g.troubled|length) == 1 }} attention</strong>
{% else %}<strong class="tone-ok">Every node is up and taking work</strong>{% endif %}
· {{ g.nodes|length }} node{{ "s" if g.nodes|length != 1 }}
· <span class="tone-{{ 'ok' if g.services_healthy == g.services_total else 'warn' }}">{{ g.services_healthy }}/{{ g.services_total }}</span> services healthy
<span class="tone-dim">· reported by {{ g.managers|join(", ") }}</span>
</p>
{% if down_nodes or g.troubled %}
<section class="card problem-card" aria-label="Needs attention">
<ul class="problem-list">
{% for n in down_nodes %}
<li>
{{ status_badge('crit', 'Node ' ~ n.status) if n.status != 'ready' else status_badge('warn', n.availability|capitalize) }}
<span>{{ n.hostname }}</span>
<span class="problem-detail">{{ n.role }} · {{ n.task_count }} task{{ "s" if n.task_count != 1 }} placed on it</span>
</li>
{% endfor %}
{% for s in g.troubled %}
<li>
{% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}{% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}{% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
<a href="/plugins/docker/service/{{ s.name }}">{{ s.name }}</a>
{% set e = s.last_error %}
<span class="problem-detail">
{{- s.running }} of {{ s.desired }} running
{%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
{%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
</span>
</li>
{% endfor %}
</ul>
</section>
{% endif %}
<div class="node-grid">
{% for n in g.nodes %}
<section class="card node-card{% if not n.ok %} problem-card{% endif %}" aria-labelledby="node-{{ n.node_id }}">
<h2 id="node-{{ n.node_id }}" class="node-card-head">
{{ status_dot('up' if n.status == 'ready' else 'down') }}
{% if n.host_id %}<a href="/hosts/{{ n.host_id }}">{{ n.hostname }}</a>{% else %}{{ n.hostname }}{% endif %}
<span class="chip">{{ "leader" if n.leader else n.role }}</span>
{% if n.availability != 'active' %}{{ status_badge('warn', n.availability|capitalize, title="Swarm places no new tasks here") }}{% endif %}
</h2>
<p class="node-card-figures">
{% if n.cores_used is not none %}
Containers use <strong>{{ fmt_cores(n.cores_used) }}</strong>{% if n.cores %} of {{ n.cores }}{% endif %}
· <strong>{{ n.mem_used|bytes }}</strong>{% if n.mem %} of {{ n.mem|bytes }}{% endif %}
{% elif n.agent %}
<span class="tone-muted">No swarm task reports figures here yet.</span>
{% else %}
<span class="tone-muted" title="Install the Steward agent on this node to see what its tasks use">No Steward agent: what runs here is Swarm's placement, without figures.</span>
{% endif %}
</p>
{% if n.tasks %}
<table class="table node-tasks">
<thead><tr><th>Service</th><th class="num">Tasks</th><th class="num">CPU</th><th class="num">Memory</th></tr></thead>
<tbody>
{% for t in n.tasks %}
<tr>
<td class="cell-name">
{%- if t.state != 'healthy' %}{{ status_dot('down' if t.state == 'down' else 'warn') }} {% endif -%}
<a href="/plugins/docker/service/{{ t.name }}" title="{{ t.name }}">{{ t.name }}</a>
</td>
<td class="num">{{ t.tasks }}</td>
<td class="num">{% if t.cores is not none %}{{ fmt_cores(t.cores) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{% if t.mem_bytes is not none %}{{ t.mem_bytes|bytes }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p class="tone-muted" style="margin:0;">No tasks placed here.</p>
{% endif %}
</section>
{% endfor %}
<div style="margin-bottom:2rem;">
<div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.75rem;flex-wrap:wrap;">
<span style="font-weight:600;font-size:0.95rem;">Swarm</span>
<span style="font-size:0.78rem;color:var(--text-muted);">
reported by {{ g.managers | length }} manager{{ 's' if g.managers | length != 1 }} · {{ g.managers | join(", ") }}
</span>
</div>
</section>
{# ── Services ─────────────────────────────────────────────────────────── #}
<div class="card-flush" style="margin-bottom:1rem;">
<table class="table">
<thead>
<tr>
<th>Service</th><th>Mode</th><th>Replicas</th><th>Image</th><th>Placement</th>
</tr>
</thead>
<tbody>
{% for s in g.services %}
<tr>
<td style="font-weight:500;font-size:0.9rem;">{{ s.name }}</td>
<td style="font-size:0.82rem;color:var(--text-muted);">{{ s.mode }}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
<span style="color:{% if s.healthy %}var(--green){% elif s.down %}var(--red){% elif s.degraded %}var(--orange){% else %}var(--text-muted){% endif %};">
{{ s.running }}/{{ s.desired }}
</span>
</td>
<td style="font-size:0.8rem;color:var(--text-muted);max-width:220px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.image or "—" }}</td>
<td style="font-size:0.78rem;color:var(--text-muted);">
{% for p in s.placement %}{{ p.node }} ({{ p.running }}){% if not loop.last %}, {% endif %}{% else %}—{% endfor %}
</td>
</tr>
{% else %}
<tr><td colspan="5" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No services in this swarm.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
{# ── Nodes ────────────────────────────────────────────────────────────── #}
<div class="card-flush">
<table class="table">
<thead>
<tr><th>Node</th><th>Role</th><th>Availability</th><th>Status</th></tr>
</thead>
<tbody>
{% for n in g.nodes %}
<tr>
<td style="font-weight:500;font-size:0.9rem;">
{{ n.hostname or n.node_id }}
{% if n.leader %}<span title="cluster leader" style="font-size:0.68rem;color:var(--accent);border:1px solid var(--accent);border-radius:3px;padding:0 0.3rem;margin-left:0.3rem;">leader</span>{% endif %}
</td>
<td style="font-size:0.82rem;color:var(--text-muted);">{{ n.role }}</td>
<td style="font-size:0.82rem;color:{% if n.availability == 'active' %}var(--text){% else %}var(--orange){% endif %};">{{ n.availability }}</td>
<td style="font-size:0.82rem;">
<span class="dot {% if n.status == 'ready' %}dot-up{% else %}dot-down{% endif %}"></span>
<span style="color:{% if n.status == 'ready' %}var(--green){% else %}var(--red){% endif %};">{{ n.status }}</span>
</td>
</tr>
{% else %}
<tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No nodes reported.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{% endfor %}
{% else %}
<div class="card empty-state">
No Swarm topology reported. A Steward host agent running on a Swarm
<strong>manager</strong> reports services, nodes and task placement here;
workers and hosts outside a swarm contribute only their own containers.
<div class="card" style="text-align:center;padding:3rem;">
<div style="color:var(--text-muted);font-size:0.9rem;">
No Swarm topology reported. A Steward host agent running on a Swarm
<strong>manager</strong> reports services, nodes, and task placement here —
workers and non-swarm hosts contribute only their local containers.
</div>
</div>
{% endif %}
{% endblock %}
plugins/docker/templates/docker/widget.html
+54 -67
View File
@@ -1,80 +1,67 @@
{# docker/widget.html — the dashboard Docker widget (#5252), in the widget-shape
order: a verdict line; the services and containers in trouble; what failed
in the last 24h, by name; then everything healthy, one line each, packed
into as many columns as the widget is wide. Resource use is the Busiest
Containers widget's, so it isn't repeated here. #}
{% from "_macros.html" import status_dot %}
{# docker/widget.html — dashboard widget: container status overview, by host #}
{% if total_count == 0 %}
<div class="w-verdict" style="color:var(--text-muted);">No containers reported yet.</div>
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No containers reported yet.</p>
{% else %}
{% set wrong = bad_services|length + bad_containers|length %}
<div class="w-verdict {{ 'tone-crit' if wrong else ('tone-warn' if failures else 'tone-ok') }}">
{%- if services_total %}{{ services_healthy }}/{{ services_total }} services healthy · {% endif -%}
{{ running_count }} other container{{ 's' if running_count != 1 }} running ·
{% if failures %}<a href="/plugins/docker/#failures" style="color:inherit;">{{ failures|length }} failed in 24h</a>{% else %}nothing failed in 24h{% endif -%}
<div style="display:flex;gap:1.25rem;margin-bottom:0.65rem;">
<div>
<span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--green);">{{ running_count }}</span>
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">running</span>
</div>
<div title="Distinct containers with a die or OOM event in the last 24h">
<span style="font-size:1.4rem;font-weight:700;line-height:1;color:{% if failed_24h %}var(--red){% else %}var(--text-muted){% endif %};">{{ failed_24h }}</span>
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">failed (24h)</span>
</div>
</div>
{% if wrong %}
<div class="w-section">Needs a look</div>
{% for s in bad_services %}
<div class="w-item">
{{ status_dot('degraded' if s.state == 'degraded' else 'down') }}
<span class="w-label" title="{{ s.name }}">{{ s.name }}
<span class="w-sub {{ 'tone-crit' if s.state != 'degraded' else 'tone-warn' }}">
{%- if s.crash_looping %}crash-looping: {{ s.failures_1h }} failed in the last hour{% else %}{{ s.running }} of {{ s.desired }} running{% endif %}
{%- if s.trouble_since %}, since {{ s.trouble_since|ago }}{% endif %}
{%- if s.last_error and (s.last_error.error or s.last_error.message) %} · {{ s.last_error.error or s.last_error.message }}{% endif -%}
</span>
</span>
<span class="w-meta" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
{# Swarm services — collapsed, each with its replicas' host chips (dashed = a
node with no Steward agent, count-only from placement). #}
{% if swarm_services %}
<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">Swarm services</div>
<div style="display:grid;gap:0.4rem;margin-bottom:0.4rem;">
{% for s in swarm_services %}
<div>
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
<span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
<span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
<span style="color:var(--text-muted);font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;">{{ s.running }}/{{ s.desired }}</span>
</div>
<div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">
{% for r in s.replicas %}
{% if r.ghost %}
<span title="no Steward agent on this node — count from swarm placement" style="font-size:0.68rem;color:var(--text-dim);background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }} ×{{ r.count }}</span>
{% else %}
<span style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }}</span>
{% endif %}
{% endfor %}
</div>
</div>
{% endfor %}
</div>
{% endfor %}
{% for c in bad_containers %}
<div class="w-item">
{{ status_dot('warn' if c.status == 'restarting' else 'down') }}
<span class="w-label" title="{{ c.name }}">
{%- if session.user_id %}<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>{% else %}{{ c.name }}{% endif %}
<span class="w-sub tone-crit">
{%- if c.health == 'unhealthy' %}unhealthy{% elif c.status == 'restarting' %}restarting{% else %}exited {{ c.exit_code }}{% if c.oom_killed %}, out of memory{% endif %}{% endif %} · {{ host_name.get(c.host_id, c.host_id) }}
</span>
</span>
</div>
{% endfor %}
{% endif %}
{% if failures %}
<div class="w-section">Failed in the last 24h</div>
{% for f in failures[:5] %}
<div class="w-item">
{{ status_dot('failed') }}
<span class="w-label" title="{{ f.name }}">{{ f.name }}
<span class="w-sub">{% if f.count > 1 %}{{ f.count }} times, last {% endif %}{{ f.at|ago }}{% if f.detail %} · {{ f.detail }}{% endif %} · {{ f.host }}</span>
</span>
</div>
{% endfor %}
{% if failures|length > 5 %}<div class="w-more">+{{ failures|length - 5 }} more · <a href="/plugins/docker/#failures">the full list</a></div>{% endif %}
{% for g in host_groups %}
{% if multi_host %}
<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">{{ g.host_name }}</div>
{% endif %}
{% if healthy %}
<div class="w-section">Healthy</div>
<div class="w-grid">
{% for h in healthy %}
<div class="w-item">
{% if h.kind == 'service' %}{% set s = h.s %}
{{ status_dot('up') }}
<span class="w-label" title="{{ s.name }} — on {{ s.replicas|map(attribute='host')|unique|join(', ') }}">{{ s.name }}
<span style="color:var(--text-dim);"> · {{ s.replicas|map(attribute='host')|unique|join(', ') }}</span></span>
<span class="w-meta" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
{% else %}{% set c = h.c %}
{{ status_dot('up' if h.kind == 'container' else 'unknown') }}
<span class="w-label" title="{{ c.name }}">
{%- if session.user_id %}<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>{% else %}{{ c.name }}{% endif %}
<span style="color:var(--text-dim);"> · {{ host_name.get(c.host_id, c.host_id) }}</span></span>
{% if h.kind == 'stopped' %}<span class="w-meta">stopped</span>{% endif %}
<div style="display:grid;gap:2px;">
{% for c in g.containers %}
<div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
<span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;">
{{ c.name }}{% if c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);"> ●</span>{% elif c.health == 'healthy' %}<span title="healthy" style="color:var(--green);"> ●</span>{% endif %}{% if c.restart_count %}<span title="restarts" style="color:var(--orange);font-size:0.7rem;"> ⟳{{ c.restart_count }}</span>{% endif %}
</a>
{% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
<span title="last exit code" style="font-size:0.7rem;color:var(--red);flex-shrink:0;">exit {{ c.exit_code }}</span>
{% endif %}
{% if c.cpu_pct is not none %}
<span style="font-size:0.72rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
{{ "%.1f" | format(c.cpu_pct) }}%
</span>
{% endif %}
</div>
{% endfor %}
</div>
{% if healthy_more %}<div class="w-more">+{{ healthy_more }} more healthy · <a href="/plugins/docker/">all of Docker</a></div>{% endif %}
{% endif %}
{% endfor %}
{% endif %}
plugins/docker/templates/docker/widget_resources.html
+23 -45
View File
@@ -1,51 +1,29 @@
{# docker/widget_resources.html — the dashboard Busiest Containers widget
(#5253, #5214): a verdict naming the busiest, then the top containers by CPU
(averaged over the last few minutes, so the order holds still). Each figure
carries its unit, beside bars of the last few hours. A row's bars are scaled
to its own peak, but never below a floor (1 core, 1 GiB by default), so a
container idling at 0.05 cores reads flat and a spike means real load. #}
{% macro cores(v) -%}{{ "%.2f"|format(v) if v < 1 else "%.1f"|format(v) }}{%- endmacro %}
{% macro bars(b, peak_text, roof_text) -%}
<span class="w-bars w-wide-only" aria-hidden="true"
title="{% if b.peak is none %}No samples in the last {{ hours }}h{% else %}Peak {{ peak_text }} in the last {{ hours }}h; bars reach the top at {{ roof_text }}{% endif %}">
{%- for h in b.bars %}<i{% if h is none %} class="gap"{% else %} style="height:{{ [h, 4]|max }}%"{% endif %}></i>{% endfor -%}
</span>
{%- endmacro %}
{# docker/widget_resources.html — dashboard widget: the busiest running containers
by CPU, each with labeled CPU + memory utilisation bars. #}
{% if not rows %}
<div class="w-verdict" style="color:var(--text-muted);">No running containers.</div>
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No running containers.</div>
{% else %}
{% set top = rows[0] %}
<div class="w-verdict">
Busiest: <strong>{{ top.label }}</strong> on {{ top.host_name }},
{% if top.usage.cores is not none %}{{ cores(top.usage.cores) }}
{%- if top.usage.cpus %} of {{ top.usage.cpus }} cores{% else %} cores{% endif %}{% else %}no CPU reading yet{% endif %}.
{% macro bar(label, pct, warn, crit) %}
<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;">
<span style="font-size:0.62rem;color:var(--text-dim);width:1.9rem;flex-shrink:0;letter-spacing:0.03em;">{{ label }}</span>
<div style="flex:1;height:6px;background:var(--bg-elevated);border-radius:3px;overflow:hidden;">
<div style="height:100%;border-radius:3px;width:{{ [pct, 100] | min }}%;
background:{% if pct > crit %}var(--red){% elif pct > warn %}var(--orange){% else %}var(--accent){% endif %};
transition:width 0.4s;"></div>
</div>
<span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;width:3rem;text-align:right;flex-shrink:0;">{{ "%.1f" | format(pct) }}%</span>
</div>
<table class="w-table">
<colgroup><col><col class="w-wide-only" style="width:7.5rem"><col class="w-barcol"><col class="w-barcol"></colgroup>
<thead><tr><th>Container</th><th class="w-wide-only w-text">Host</th>
<th title="Cores, averaged over the last {{ minutes }} minutes; bars: the last {{ hours }}h">CPU<span class="w-wide-only"> · last {{ hours }}h</span></th>
<th title="In use now; bars: the last {{ hours }}h">Memory<span class="w-wide-only"> · last {{ hours }}h</span></th></tr></thead>
<tbody>
{% endmacro %}
<div style="display:grid;gap:0.6rem;">
{% for r in rows %}
{% set u = r.usage %}
<tr>
<td class="w-name" title="{{ r.c.name }}">
{%- if session.user_id %}<a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}">{{ r.label }}</a>{% else %}{{ r.label }}{% endif -%}
</td>
<td class="w-wide-only w-text" title="{{ r.host_name }}">{{ r.host_name }}</td>
<td class="tone-{{ status_class(u.cpu_share, 'cpu') if u.cpus and u.cores is not none else 'dim' }}"
{% if u.cpus and u.cores is not none %}title="{{ '%.1f'|format(u.cpu_share) }}% of {{ r.host_name }}'s {{ u.cpus }} cores"{% endif %}>
{{- bars(r.cpu_bars, (cores(r.cpu_bars.peak) ~ " cores") if r.cpu_bars.peak is not none else "", cores(r.cpu_bars.roof) ~ " cores") -}}
<span class="w-fig">{% if u.cores is none %}—{% else %}{{ cores(u.cores) }} cores{% endif %}</span>
</td>
<td class="tone-{{ status_class(u.mem_pct, 'mem') if u.mem_of and u.mem_bytes is not none else 'dim' }}"
{% if u.mem_pct is not none and u.mem_of and u.mem_bytes is not none %}title="{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}"{% endif %}>
{{- bars(r.mem_bars, r.mem_bars.peak|bytes if r.mem_bars.peak is not none else "", r.mem_bars.roof|bytes) -}}
<span class="w-fig">{% if u.mem_bytes is none %}—{% else %}{{ u.mem_bytes|bytes }}{% endif %}</span>
</td>
</tr>
{% set c = r.c %}
<div>
<div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;">
{{ c.name }}{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
</div>
{% if c.cpu_pct is not none %}{{ bar("CPU", c.cpu_pct, 50, 80) }}{% endif %}
{% if c.mem_pct is not none %}{{ bar("MEM", c.mem_pct, 70, 90) }}{% endif %}
</div>
{% endfor %}
</tbody>
</table>
<div class="w-note w-wide-only">Bars: {{ (hours * 60 // 24) }}-minute slices, oldest left; each row up to its own peak, at least {{ cores(cpu_floor) }} core{{ 's' if cpu_floor != 1 }} and {{ mem_floor|bytes }}.</div>
</div>
{% endif %}
plugins/host_agent/__init__.py
-23
View File
@@ -21,26 +21,3 @@ def get_scheduled_tasks() -> list:
def get_blueprint():
from .routes import host_agent_bp
return host_agent_bp
def register_mcp_tools(register) -> None:
from .mcp_tools import register_tools
register_tools(register)
async def mcp_overview(db) -> list[dict]:
from .mcp_tools import mcp_overview as findings
return await findings(db)
def get_alert_metrics() -> dict[str, list[str]]:
"""Host metrics ingest records for alert rules (resource = the host name;
disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
comes from the scheduler, not the agent: "agent offline" is > 180.
clock_off_secs is how far the host's clock is from Steward's, either way
(agent ≥ 1.12.0): "clock drifting" is > 30."""
return {"host_agent": [
"cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
"disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m",
"uptime_secs", "seconds_since_seen", "clock_off_secs",
]}
plugins/host_agent/agent.py
+59 -473
View File
@@ -17,10 +17,10 @@ import time
import urllib.error
import urllib.request
from collections import deque
from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
AGENT_VERSION = "1.12.0"
AGENT_VERSION = "1.6.0"
# Default path to the local Docker Engine socket. Overridable via the
# `docker_socket` config key; collection is silently skipped if it's absent or
@@ -34,10 +34,7 @@ class ConfigError(Exception):
REQUIRED_KEYS = ("url", "token")
INT_KEYS = ("interval_seconds",)
LIST_KEYS = ("mounts", "docker_log_exclude")
# Truthy strings for bool keys; anything else (incl. empty) is False.
BOOL_KEYS = ("docker_logs_enabled",)
_BOOL_TRUE = ("1", "true", "yes", "on")
LIST_KEYS = ("mounts",)
def read_config(path: str) -> dict:
@@ -61,8 +58,6 @@ def read_config(path: str) -> dict:
raise ConfigError(f"{path}:{lineno}: {key} must be int")
elif key in LIST_KEYS:
cfg[key] = [v.strip() for v in value.split(",") if v.strip()]
elif key in BOOL_KEYS:
cfg[key] = value.lower() in _BOOL_TRUE
else:
cfg[key] = value
except FileNotFoundError:
@@ -74,10 +69,6 @@ def read_config(path: str) -> dict:
cfg.setdefault("interval_seconds", 30)
cfg.setdefault("docker_socket", DEFAULT_DOCKER_SOCKET)
# Container-log collection is on by default (operator preference); a host can
# opt out with `docker_logs_enabled = false` or thin it with docker_log_exclude.
cfg.setdefault("docker_logs_enabled", True)
cfg.setdefault("docker_log_exclude", [])
return cfg
@@ -91,10 +82,6 @@ OS_RELEASE_PATH = "/etc/os-release"
NETDEV_PATH = "/proc/net/dev"
DISKSTATS_PATH = "/proc/diskstats"
HWMON_DIR = "/sys/class/hwmon"
# Where the kernel lists virtual network devices (bridges, veth, docker0,
# vxlan, tun, VLANs, bonds) and the block devices (a disk has a `device`).
SYS_VIRTUAL_NET_DIR = "/sys/devices/virtual/net"
SYS_BLOCK_DIR = "/sys/block"
PSI_DIR = "/proc/pressure"
# Partition names to drop from disk I/O (we report whole-disk throughput only).
@@ -263,34 +250,10 @@ def default_mounts() -> list[str]:
return mounts or ["/"]
def _physical_only(found: dict, is_virtual) -> dict:
"""The entries the kernel says are physical, so a host total counts each
byte once (a bridge's or an LVM volume's traffic is also counted on the
NIC or disk beneath it). When /sys calls every one of them virtual (an
agent inside LXC, where eth0 is a veth), they are all kept: a host
reporting no traffic at all would be wrong, not filtered."""
physical = {k: v for k, v in found.items() if not is_virtual(k)}
return physical or found
def _virtual_net(name: str) -> bool:
"""Whether the kernel lists this interface as virtual; False if /sys can't say."""
return os.path.isdir(SYS_VIRTUAL_NET_DIR) and os.path.exists(
os.path.join(SYS_VIRTUAL_NET_DIR, name))
def _virtual_disk(name: str) -> bool:
"""Whether this block device has no disk behind it (dm, md, zram, nbd);
False if /sys can't say."""
return os.path.isdir(SYS_BLOCK_DIR) and not os.path.exists(
os.path.join(SYS_BLOCK_DIR, name, "device"))
def collect_net_raw() -> dict:
"""Return {iface: (rx_bytes, tx_bytes)} cumulative counters from /proc/net/dev.
Physical interfaces only (see _physical_only); loopback and veth* are
skipped by name too, for a host whose /sys can't be read. Counters are
Skips loopback and veth* (container) interfaces to cut noise. Counters are
monotonic; per-second rates are derived from deltas in build_sample().
"""
out: dict = {}
@@ -312,15 +275,14 @@ def collect_net_raw() -> dict:
out[name] = (int(f[0]), int(f[8])) # rx_bytes, tx_bytes
except ValueError:
continue
return _physical_only(out, _virtual_net)
return out
def collect_diskio_raw() -> dict:
"""Return {device: (read_bytes, write_bytes)} cumulative from /proc/diskstats.
Physical whole disks only (see _physical_only): loop/ram/sr/fd and
partitions are skipped by name too. Sectors are 512 bytes. Counters are
monotonic; rates derived from deltas.
Whole disks only — loop/ram/sr/fd and partitions are skipped. Sectors are
512 bytes. Counters are monotonic; rates derived from deltas.
"""
out: dict = {}
try:
@@ -338,7 +300,7 @@ def collect_diskio_raw() -> dict:
out[name] = (int(f[5]) * 512, int(f[9]) * 512) # sectors_read, sectors_written
except ValueError:
continue
return _physical_only(out, _virtual_disk)
return out
def collect_temps() -> list:
@@ -440,24 +402,6 @@ def _rates(cur: dict, prev: dict, dt: float) -> dict:
# when that header is present rather than assume Content-Length.
DOCKER_API_TIMEOUT = 5.0
# All of one sample's per-container Docker work (inspect + stats) must finish
# inside this; past it the sample says Docker did not answer in time rather
# than holding up the host's own metrics (#5154).
DOCKER_DEADLINE_SECONDS = 20.0
# /system/df walks every image, container and volume: asked this often, not
# every sample. Steward keeps the last answer between reports.
DOCKER_DF_INTERVAL_SECONDS = 600.0
# Container-log collection (m79). Logs ride in the same push as metrics, so one
# interval's batch is capped to keep a chatty container from bloating a POST (and
# the backoff ring): once the cap is hit a marker line is emitted and collection
# stops — the deferred lines come on the next interval. On a container's first
# interval we seed from a short tail, then switch to an incremental since-cursor
# kept per container in the agent's rate-state.
DOCKER_LOG_BATCH_MAX_BYTES = 262144 # 256 KiB of log text per push, all containers
DOCKER_LOG_FIRST_TAIL = 50 # lines seeded on a container's first interval
DOCKER_LOG_FINAL_TAIL = 200 # lines read once from a container found already crashed
DOCKER_LOG_FINAL_WINDOW_SECONDS = 600 # ... when it stopped this recently
def _dechunk(body: bytes) -> bytes:
@@ -601,18 +545,6 @@ def _docker_inspect_fields(insp: dict):
return health, restart_count, exit_code, oom_killed
def _docker_started_at(insp: dict, key: str = "StartedAt"):
"""State.StartedAt as ISO-8601, or None for a container never started
(Docker's zero time) or an inspect that failed. When the container last
started, not when it was created: a restarted container's uptime counts
from its restart. key="FinishedAt" reads when it last stopped."""
raw = (((insp or {}).get("State") or {}).get(key) or "")
if not raw or raw.startswith("0001-"):
return None
ts = _parse_log_ts(raw)
return ts.isoformat() if ts else None
def _docker_net_io(stats: dict):
"""Return cumulative (net_rx, net_tx, blk_read, blk_write) bytes from stats.
@@ -646,7 +578,7 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
# Inspect every container (incl. stopped) — exit codes + restart counts only
# live here, and stopped containers are exactly where exit_code matters.
health = exit_code = insp = None
health = exit_code = None
restart_count = 0
oom_killed = False
try:
@@ -666,10 +598,11 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
except (OSError, ValueError):
pass
started_at = _docker_started_at(insp)
# When it last stopped (a running container's is its previous stop, or
# None): dates a die event, and tells the log collector a stop is new.
finished_at = None if state == "running" else _docker_started_at(insp, "FinishedAt")
created = c.get("Created")
started_at = (
datetime.fromtimestamp(created, tz=timezone.utc).isoformat()
if isinstance(created, (int, float)) else None
)
record = {
"name": name,
"container_id": cid[:12],
@@ -681,7 +614,6 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
"mem_pct": mem_pct,
"ports": _docker_ports(c.get("Ports", [])),
"started_at": started_at,
"finished_at": finished_at,
"health": health,
"restart_count": restart_count,
"exit_code": exit_code,
@@ -695,276 +627,27 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
return record
def _docker_failure(socket_path: str, exc: Exception) -> tuple:
"""(status, detail) for a failed Docker call: "unavailable" when there is no
Docker here to ask, "error" when there is one and the call went wrong."""
if isinstance(exc, FileNotFoundError):
return "unavailable", "no Docker socket at " + socket_path
if isinstance(exc, ConnectionRefusedError):
return "unavailable", "Docker is not accepting connections on " + socket_path
if isinstance(exc, PermissionError):
return "error", "permission denied on " + socket_path
if isinstance(exc, socket.timeout):
return "error", "Docker did not answer within %.0f s" % DOCKER_API_TIMEOUT
return "error", (str(exc) or type(exc).__name__)[:200]
def collect_docker(socket_path: str) -> list:
"""Per-container state from the local Docker socket, or [] if unavailable.
def collect_docker(socket_path: str) -> tuple:
"""(status, detail, containers) from the local Docker socket.
status is "ok" when Docker answered (containers may then be [] on a host
with none, which tells Steward to clear its list), "unavailable" when there
is no Docker here (no socket, or nothing listening), and "error" when Docker
is here but the listing failed. detail says why, for the last two. A
Docker-less host still needs no configuration: it reports "unavailable".
Absent socket / permission denied / non-Docker endpoint all yield [] — the
agent silently reports no containers rather than erroring, so a Docker-less
host needs no configuration.
"""
try:
containers = _docker_request(socket_path, "/containers/json?all=true")
except (OSError, ValueError) as exc:
return _docker_failure(socket_path, exc) + ([],)
if not isinstance(containers, list):
return "error", "unexpected answer from /containers/json", []
if not containers:
return "ok", None, []
except (OSError, ValueError):
return []
if not isinstance(containers, list) or not containers:
return []
# Each container needs an inspect (+ a stats read if running), and the stats
# call blocks ~1s while the daemon computes the cpu delta. Done serially that
# stretches the whole sample on a busy host, so fan the per-container work out
# over a small bounded thread pool (I/O-bound → threads are enough).
# A deadline on the whole fan-out: one wedged container must not stall the
# sample. Unfinished work is reported as Docker not answering in time, not
# as a partial list: a container left out would read as removed, and one
# without its inspect as restarted or changed health.
ex = ThreadPoolExecutor(max_workers=min(8, len(containers)))
try:
futures = [ex.submit(_collect_one_container, socket_path, c) for c in containers]
done, pending = wait_futures(futures, timeout=DOCKER_DEADLINE_SECONDS)
finally:
try:
ex.shutdown(wait=False, cancel_futures=True)
except TypeError: # Python < 3.9: no cancel_futures
ex.shutdown(wait=False)
if pending:
return ("error", "%d of %d containers not read within %g s"
% (len(pending), len(containers), DOCKER_DEADLINE_SECONDS), [])
return "ok", None, [f.result() for f in futures]
# ─── container logs (m79) ─────────────────────────────────────────────────────
#
# The logs endpoint returns a raw multiplexed byte stream, not JSON, so it needs
# a sibling of _docker_request that hands back bytes. Non-TTY containers frame
# stdout/stderr with an 8-byte header (stream byte + 4-byte big-endian length);
# TTY containers emit unframed bytes. We request timestamps=1, so every line is
# prefixed with an RFC3339Nano timestamp we parse for the since-cursor.
_LOG_STREAM_NAMES = {0: "stdout", 1: "stdout", 2: "stderr"}
def _docker_request_raw(socket_path: str, path: str,
timeout: float = DOCKER_API_TIMEOUT) -> bytes:
"""GET `path` from the Docker API over the Unix socket; return the raw body.
Sibling of `_docker_request` for non-JSON endpoints (container logs). Reuses
the connect / send / de-chunk scaffolding; raises OSError on any transport
problem or non-2xx status so callers can silent-skip. A 200 with an empty
body (no new lines since the cursor) returns b"".
"""
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(timeout)
try:
sock.connect(socket_path)
req = (
"GET " + path + " HTTP/1.1\r\n"
"Host: docker\r\n"
"Accept: application/octet-stream\r\n"
"Connection: close\r\n"
"\r\n"
)
sock.sendall(req.encode("ascii"))
chunks = []
while True:
buf = sock.recv(65536)
if not buf:
break
chunks.append(buf)
finally:
sock.close()
raw = b"".join(chunks)
head, _, body = raw.partition(b"\r\n\r\n")
header_text = head.decode("latin-1")
status_line = header_text.split("\r\n", 1)[0]
parts = status_line.split(None, 2)
status = int(parts[1]) if len(parts) >= 2 and parts[1].isdigit() else 0
if not (200 <= status < 300):
raise OSError(f"docker API {path} returned {status}")
if "transfer-encoding: chunked" in header_text.lower():
body = _dechunk(body)
return body
def _demux_docker_logs(raw: bytes) -> list:
"""Split a Docker logs stream into [(stream_name, payload_bytes), …].
Non-TTY containers multiplex with an 8-byte frame header (stream byte in
{0,1,2} + 4-byte big-endian length). The stream is treated as framed only if
the whole buffer parses as clean back-to-back frames; anything else (a TTY
container's raw stream, or a malformed header) falls back to a single stdout
blob so no bytes are lost.
"""
if not raw:
return []
frames = []
i, n = 0, len(raw)
while i + 8 <= n:
stream = raw[i]
if stream > 2 or raw[i + 1] or raw[i + 2] or raw[i + 3]:
break # not a valid frame header → not framed
size = int.from_bytes(raw[i + 4:i + 8], "big")
if i + 8 + size > n:
break # frame overruns the buffer → not framed
frames.append((_LOG_STREAM_NAMES.get(stream, "stdout"),
raw[i + 8:i + 8 + size]))
i += 8 + size
if frames and i == n:
return frames
return [("stdout", raw)]
def _parse_log_ts(token: str):
"""Parse a Docker RFC3339Nano timestamp token into a datetime, or None.
Normalises `Z` → `+00:00` and trims Docker's nanosecond precision to the
microseconds datetime.fromisoformat accepts (stdlib, Python 3.8+ safe).
"""
t = token.strip()
if not t:
return None
if t.endswith("Z"):
t = t[:-1] + "+00:00"
m = re.match(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(\.\d+)?(.*)$", t)
if m:
frac = m.group(2) or ""
if len(frac) > 7: # "." + up to 6 fractional digits
frac = frac[:7]
t = m.group(1) + frac + m.group(3)
try:
return datetime.fromisoformat(t)
except ValueError:
return None
def _parse_container_logs(raw: bytes) -> list:
"""Demux + line-split a logs stream → [(dt|None, stream, line), …] in order.
Each line is timestamp-prefixed (we request timestamps=1); an unparseable
prefix yields dt=None with the full raw line kept.
"""
out = []
for stream, payload in _demux_docker_logs(raw):
text = payload.decode("utf-8", "replace")
for raw_line in text.split("\n"):
if not raw_line:
continue
token, _, msg = raw_line.partition(" ")
dt = _parse_log_ts(token)
if dt is not None:
out.append((dt, stream, msg))
else:
out.append((None, stream, raw_line))
return out
def collect_docker_logs(socket_path: str, containers: list, state: dict,
exclude=None,
max_bytes: int = DOCKER_LOG_BATCH_MAX_BYTES,
now: float | None = None) -> list:
"""New container-log lines since the last interval.
Keeps a per-container since-cursor in `state["docker_log_cursors"]`
(name → unix ts) so each interval fetches only what's new; a container's
first interval seeds from a short tail. A running container is read every
interval. A stopped one is read once per stop (`finished_at`, remembered in
`state["docker_log_finals"]`), so the lines it wrote on its way down reach
Steward (#5049): since its cursor if it was being read, or, for one first
seen already stopped with a non-zero exit in the last
DOCKER_LOG_FINAL_WINDOW_SECONDS, its last DOCKER_LOG_FINAL_TAIL lines. That
is how a task that crashes between two samples is explained by its own
output. Both maps forget a container once it is no longer listed.
The whole batch is capped at `max_bytes` of log text — once exceeded a
truncation marker is appended and collection stops (logs must never balloon
a push). `exclude` container names are skipped entirely (a per-host
bandwidth opt-out). Best-effort: a per-container transport error just
contributes nothing this interval.
Returns [{"container", "stream", "ts", "line"}, …].
"""
exclude_set = set(exclude or ())
cursors = state.setdefault("docker_log_cursors", {})
finals = state.setdefault("docker_log_finals", {})
now = time.time() if now is None else now
listed = {c.get("name"): c for c in (containers or []) if c.get("name")}
# Forget containers no longer listed so state can't grow without bound
# over the agent's lifetime.
for book in (cursors, finals):
for gone in [k for k in book if k not in listed]:
del book[gone]
out: list = []
total = 0
truncated = False
for name, c in listed.items():
if name in exclude_set:
continue
prev = cursors.get(name)
base = f"/containers/{name}/logs?stdout=1&stderr=1&timestamps=1"
if c.get("status") == "running":
# since accepts a fractional Unix ts; we still dedup the boundary line.
path = (f"{base}&tail={DOCKER_LOG_FIRST_TAIL}" if prev is None
else f"{base}&since={prev:.9f}")
else:
stopped = c.get("finished_at")
if not stopped or finals.get(name) == stopped:
continue
if prev is not None:
path = f"{base}&since={prev:.9f}"
else:
at = _parse_log_ts(stopped)
if (c.get("exit_code") in (None, 0) or at is None
or now - at.timestamp() > DOCKER_LOG_FINAL_WINDOW_SECONDS):
continue
path = f"{base}&tail={DOCKER_LOG_FINAL_TAIL}"
finals[name] = stopped
try:
raw = _docker_request_raw(socket_path, path)
except (OSError, ValueError):
continue
newest = prev
for dt, stream, line in _parse_container_logs(raw):
ts_unix = dt.timestamp() if dt is not None else None
if prev is not None and ts_unix is not None and ts_unix <= prev:
continue # boundary line already sent last interval
if total >= max_bytes:
truncated = True
break
out.append({"container": name, "stream": stream,
"ts": dt.isoformat() if dt is not None else None,
"line": line})
total += len(line)
if ts_unix is not None and (newest is None or ts_unix > newest):
newest = ts_unix
if newest is not None:
cursors[name] = newest
if truncated:
break
if truncated:
out.append({"container": "_steward", "stream": "stderr",
"ts": datetime.now(timezone.utc).isoformat(),
"line": (f"[steward] log batch truncated at {max_bytes} bytes; "
"remaining lines deferred to the next interval")})
return out
workers = min(8, len(containers))
with ThreadPoolExecutor(max_workers=workers) as ex:
return list(ex.map(lambda c: _collect_one_container(socket_path, c), containers))
# ─── swarm (manager-only) ────────────────────────────────────────────────────
@@ -981,46 +664,14 @@ def _swarm_is_manager(info: dict) -> bool:
return bool((info or {}).get("Swarm", {}).get("ControlAvailable"))
# Newest tasks kept per service. Swarm retains about 5 per slot by default, so
# this covers a replicated service's recent history and a crash loop's deaths
# without shipping a global service's whole fleet every sample.
SWARM_TASKS_PER_SERVICE = 10
_TASK_TEXT_CHARS = 300
def _swarm_task(t: dict) -> dict:
"""One task as Steward stores it: where it ran, what Swarm wants of it,
what it is doing, and why it stopped (Err/Message, exit code)."""
status = t.get("Status") or {}
cstatus = status.get("ContainerStatus") or {}
exit_code = cstatus.get("ExitCode")
return {
"task_id": t.get("ID", "") or "",
"slot": t.get("Slot"),
"node_id": t.get("NodeID", "") or "",
"desired_state": (t.get("DesiredState") or "").lower(),
"state": (status.get("State") or "").lower(),
"message": (status.get("Message") or "")[:_TASK_TEXT_CHARS],
"error": (status.get("Err") or "")[:_TASK_TEXT_CHARS],
"exit_code": exit_code if isinstance(exit_code, int) else None,
"at": status.get("Timestamp") or t.get("UpdatedAt") or "",
}
def _swarm_services(services: list, tasks: list) -> list:
"""Roll desired-vs-running replicas per service up from the task list.
Replica health isn't on the service object — it's derived by counting tasks:
`running` = tasks currently in state running; `desired` = the service's
configured replica count (replicated) or the count of tasks the scheduler
wants running (global, which has no fixed number). Each service also
carries its newest tasks (`tasks`), so Steward can tell a task Swarm
replaced from one that crashed, and say why the last one failed.
wants running (global, which has no fixed number).
"""
tasks_by_svc: dict = {}
for t in (tasks or []):
if t.get("ServiceID"):
tasks_by_svc.setdefault(t["ServiceID"], []).append(_swarm_task(t))
running_by_svc: dict = {}
desired_by_svc: dict = {} # only used for global-mode services
placement: dict = {} # svc_id -> {node_id: running_count}
@@ -1060,8 +711,6 @@ def _swarm_services(services: list, tasks: list) -> list:
# sees every task, even ones whose containers live on other hosts).
"placement": [{"node_id": n, "running": cnt}
for n, cnt in sorted(placement.get(sid, {}).items())],
"tasks": sorted(tasks_by_svc.get(sid, []), key=lambda t: t["at"],
reverse=True)[:SWARM_TASKS_PER_SERVICE],
})
return out
@@ -1085,28 +734,26 @@ def _swarm_nodes(nodes: list) -> list:
return out
def collect_swarm(socket_path: str) -> tuple:
"""(is_manager, topology) from the local Docker socket.
def collect_swarm(socket_path: str):
"""Swarm topology from a manager node, or None on workers / non-swarm hosts.
Self-detects manager via `/info` (one cheap call); only then queries the
manager-only endpoints. is_manager is None when /info itself failed, so a
transient error is never mistaken for a node that stopped being a manager;
False tells Steward to drop the swarm rows this host reported as one.
topology is None unless this is a manager and every call answered.
manager-only endpoints. Any transport failure degrades to None, same silent
contract as collect_docker — a non-swarm host adds nothing to the payload.
"""
try:
info = _docker_request(socket_path, "/info")
except (OSError, ValueError):
return None, None
return None
if not _swarm_is_manager(info):
return False, None
return None
try:
services = _docker_request(socket_path, "/services")
tasks = _docker_request(socket_path, "/tasks")
nodes = _docker_request(socket_path, "/nodes")
except (OSError, ValueError):
return True, None
return True, {
return None
return {
"services": _swarm_services(
services if isinstance(services, list) else [],
tasks if isinstance(tasks, list) else []),
@@ -1205,17 +852,13 @@ class RingBuffer:
def build_sample(mounts: list[str], state: dict,
docker_socket: str = DEFAULT_DOCKER_SOCKET, *,
docker_logs_enabled: bool = True,
docker_log_exclude=None) -> dict:
docker_socket: str = DEFAULT_DOCKER_SOCKET) -> dict:
"""Collect one full sample. Partial samples allowed if a collector fails.
`state` carries the previous network/disk counters + monotonic timestamp so
throughput rates can be derived from deltas (and the per-container log
since-cursors); it is mutated in place. `docker_socket` is probed
best-effort — the `docker` key is omitted entirely when no containers are
found, so non-Docker hosts add nothing to the payload. `docker_logs_enabled`
/ `docker_log_exclude` gate incremental container-log collection.
throughput rates can be derived from deltas; it is mutated in place.
`docker_socket` is probed best-effort — the `docker` key is omitted entirely
when no containers are found, so non-Docker hosts add nothing to the payload.
"""
sample: dict = {"ts": datetime.now(timezone.utc).isoformat()}
try:
@@ -1270,33 +913,26 @@ def build_sample(mounts: list[str], state: dict,
]
state["net"], state["disk"], state["mono"] = net_raw, disk_raw, now_mono
# Docker's status rides every sample, so Steward can tell "no containers"
# (status ok, docker: []) from "couldn't ask" (unavailable / error).
try:
docker_status, docker_detail, docker = collect_docker(docker_socket)
except Exception as exc:
docker_status, docker_detail, docker = "error", (str(exc) or type(exc).__name__)[:200], []
sample["docker_status"] = docker_status
if docker_detail:
sample["docker_detail"] = docker_detail
if docker_status == "ok":
docker = collect_docker(docker_socket)
except Exception:
docker = []
if docker:
sample["docker"] = docker
# Swarm topology is only emitted by managers. A node whose /info says it
# is not a manager says so, so Steward can expire what it once reported.
# Swarm topology is only emitted by managers (collect_swarm returns None
# everywhere else), so the key is absent on workers / non-swarm hosts.
try:
is_manager, swarm = collect_swarm(docker_socket)
swarm = collect_swarm(docker_socket)
except Exception:
is_manager, swarm = None, None
swarm = None
if swarm is not None:
sample["swarm"] = swarm
elif is_manager is False:
sample["swarm_manager"] = False
# Image/disk usage — host-level docker fact (one /system/df call), only
# when Docker answered.
if docker_status == "ok" and now_mono - state.get("df_mono", float("-inf")) >= DOCKER_DF_INTERVAL_SECONDS:
state["df_mono"] = now_mono
# Image/disk usage — host-level docker fact (one /system/df call), omitted on
# Docker-less hosts. Only meaningful when there's docker here at all, so skip
# the call entirely when collect_docker found nothing.
if docker:
try:
disk = collect_disk_usage(docker_socket)
except Exception:
@@ -1304,29 +940,14 @@ def build_sample(mounts: list[str], state: dict,
if disk is not None:
sample["docker_disk"] = disk
# Container logs (m79): incremental per-container tail folded into the same
# push as metrics. Skipped when there are no containers or logging is off;
# the key is omitted when nothing new arrived this interval.
if docker and docker_logs_enabled:
try:
logs = collect_docker_logs(docker_socket, docker, state,
exclude=docker_log_exclude)
except Exception:
logs = []
if logs:
sample["docker_logs"] = logs
return sample
def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict:
# sent_at is this host's clock at the moment of sending: Steward compares
# it with its own on arrival to say how far apart the two clocks are.
return {
"agent_version": AGENT_VERSION,
"hostname": hostname,
"metadata": metadata,
"sent_at": datetime.now(timezone.utc).isoformat(),
"samples": samples,
}
@@ -1334,32 +955,7 @@ def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict:
# ─── POST + backoff ──────────────────────────────────────────────────────────
# Below Steward's 180 s staleness window (host_agent stale_after_seconds): once
# Steward is reachable again a host reports within a minute, instead of reading
# as stale for up to five (#5154; lesson: a check-in inherits its call's backoff).
BACKOFF_CAP = 60
# Samples in one POST: the server writes at most this many (routes.py
# MAX_SAMPLES_PER_INGEST), so a backlog drains a few per post, oldest first,
# rather than in one post too big to write before the timeout.
MAX_SAMPLES_PER_POST = 5
def _drop_logs(sample: dict) -> dict:
"""Strip container logs from a sample before it's buffered for retry.
Logs are the one payload we never carry across a backoff — they'd bloat the
ring buffer and go stale — while metrics are kept so an outage doesn't lose
them. The since-cursor already advanced when the logs were collected, so the
dropped lines are simply not re-sent (accepted loss during an outage).
"""
sample.pop("docker_logs", None)
return sample
def split_backlog(buffered: list, limit: int = MAX_SAMPLES_PER_POST) -> tuple:
"""(the buffered samples to send with this one, the ones held for later):
the oldest limit-1, leaving room for the new sample."""
return buffered[:limit - 1], buffered[limit - 1:]
BACKOFF_CAP = 300
def next_backoff(current: int) -> int:
@@ -1426,8 +1022,6 @@ def main_loop(conf_path: str) -> int:
hostname = cfg.get("hostname") or socket.gethostname()
mounts = cfg.get("mounts") or default_mounts()
docker_socket = cfg.get("docker_socket") or DEFAULT_DOCKER_SOCKET
docker_logs_enabled = cfg.get("docker_logs_enabled", True)
docker_log_exclude = cfg.get("docker_log_exclude") or []
buffer = RingBuffer(maxlen=20)
backoff = 0
# Carries previous net/disk counters + monotonic ts for rate computation.
@@ -1442,32 +1036,24 @@ def main_loop(conf_path: str) -> int:
cfg = read_config(conf_path)
mounts = cfg.get("mounts") or default_mounts()
docker_socket = cfg.get("docker_socket") or DEFAULT_DOCKER_SOCKET
docker_logs_enabled = cfg.get("docker_logs_enabled", True)
docker_log_exclude = cfg.get("docker_log_exclude") or []
metadata = collect_metadata() # refresh host_ip/distro on reload
_log("INFO", "config reloaded")
except ConfigError as e:
_log("ERROR", f"reload failed: {e}")
_reload_requested = False
sample = build_sample(mounts, rate_state, docker_socket,
docker_logs_enabled=docker_logs_enabled,
docker_log_exclude=docker_log_exclude)
sample = build_sample(mounts, rate_state, docker_socket)
buffered = buffer.drain()
send, held = split_backlog(buffered)
payload = build_payload(
samples=send + [sample],
samples=buffered + [sample],
hostname=hostname,
metadata=metadata,
)
ok, status = post_payload(cfg["url"], cfg["token"], payload)
if ok:
for s in held:
buffer.push(s)
if send:
_log("INFO", f"flushed {len(send)} buffered samples"
+ (f", {len(held)} still held" if held else ""))
if buffered:
_log("INFO", f"flushed {len(buffered)} buffered samples")
backoff = 0
sleep_for = cfg["interval_seconds"]
else:
@@ -1475,12 +1061,12 @@ def main_loop(conf_path: str) -> int:
_log("ERROR", "server rejected payload (400) — dropping sample")
elif status == 401:
_log("ERROR", "token rejected (401) — check config + UI")
buffer.push(_drop_logs(sample))
buffer.push(sample)
else:
_log("WARN", f"POST failed (status={status}); buffering")
for s in buffered:
buffer.push(_drop_logs(s))
buffer.push(_drop_logs(sample))
buffer.push(s)
buffer.push(sample)
backoff = next_backoff(backoff)
sleep_for = backoff
plugins/host_agent/mcp_tools.py
-81
View File
@@ -1,81 +0,0 @@
"""The host_agent plugin's MCP tools and its get_overview findings."""
from __future__ import annotations
from datetime import timedelta
from sqlalchemy import select
from steward.hosts import service as host_svc
from steward.mcp._context import mcp_app
from steward.mcp.tools._shape import age_s, db_session, iso, now_utc
from steward.mcp.tools.overview import finding
from steward.models.hosts import Host
from steward.models.users import UserRole
from .models import HostAgentRegistration
# A registration this old that has never checked in is a failed install, not one in progress.
PENDING_GRACE = timedelta(minutes=30)
def agent_state(reg, now, stale_after_s: int) -> str:
"""The agent's state: reporting, stale (stopped checking in) or pending (never has)."""
if reg.last_seen_at is None:
return "pending"
return "reporting" if (now - reg.last_seen_at).total_seconds() <= stale_after_s else "stale"
async def _agents(db):
return (await db.execute(
select(HostAgentRegistration, Host)
.join(Host, Host.id == HostAgentRegistration.host_id)
.order_by(Host.name)
)).all()
async def list_agents() -> dict:
"""Every host agent Steward has registered, with its version and whether it is reporting.
`state` is "reporting", "stale" (it checked in before but its last
check-in is older than the stale window, `stale_after_s`) or "pending"
(registered, never checked in, usually an install that is still running
or that failed). Each row carries the agent version, kernel, distro, arch
and the IP the agent reports. get_host shows an agent's latest metrics.
"""
now = now_utc()
stale_after = host_svc.agent_stale_after_seconds(mcp_app().config)
async with db_session() as db:
rows = await _agents(db)
agents = [
{
"host_id": h.id, "host_name": h.name,
"state": agent_state(reg, now, stale_after),
"last_seen_at": iso(reg.last_seen_at), "last_seen_age_s": age_s(reg.last_seen_at, now),
"agent_version": reg.agent_version, "kernel": reg.kernel, "distro": reg.distro,
"arch": reg.arch, "reported_ip": reg.host_ip,
"registered_at": iso(reg.created_at),
}
for reg, h in rows
]
counts = {s: sum(1 for a in agents if a["state"] == s) for s in ("reporting", "stale", "pending")}
return {"stale_after_s": stale_after, "counts": counts, "agents": agents, "total": len(agents)}
async def mcp_overview(db) -> list[dict]:
"""Agents registered more than PENDING_GRACE ago that have never checked in.
Agents that stopped reporting are already flagged by the core overview
from their metrics, so they are not repeated here.
"""
now = now_utc()
return [
finding("warning", "agent_never_reported", h.name,
"host agent was registered but has never checked in; the install may have failed",
since=reg.created_at, ids={"host_id": h.id}, next="list_agents()")
for reg, h in await _agents(db)
if reg.last_seen_at is None and now - reg.created_at > PENDING_GRACE
]
def register_tools(register) -> None:
register(list_agents, role=UserRole.viewer, read_only=True)
plugins/host_agent/metrics_query.py
+23 -7
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
from datetime import datetime, timedelta, timezone
from sqlalchemy import func, select
from sqlalchemy import func, or_, select
from steward.core.time_range import bucket_seconds
from steward.models.metrics import PluginMetric, PluginMetricHourly
@@ -27,14 +27,30 @@ HISTORY_METRICS = (
async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]:
"""{resource_name: {metric: value}} — latest sample per (resource, metric) for
a host + its sub-resources, from plugin_metric_latest (#5208). A host whose
agent has gone quiet keeps its last values until they age out of the raw
window; the caller decides staleness from the registration's last_seen_at.
"""
from steward.core.metric_series import latest_values
a host + its sub-resources.
DISTINCT ON picks the newest row per group in one index-ordered pass over
ix_plugin_metrics_module_resource_metric_recorded, instead of a GROUP-BY-max
subquery self-joined back to the table (two passes over the whole history).
"""
rows = (await session.execute(
select(PluginMetric)
.where(
PluginMetric.source_module == SOURCE_MODULE,
or_(
PluginMetric.resource_name == host_name,
PluginMetric.resource_name.like(host_name + ":%"),
),
)
.distinct(PluginMetric.resource_name, PluginMetric.metric_name)
.order_by(
PluginMetric.resource_name,
PluginMetric.metric_name,
PluginMetric.recorded_at.desc(),
)
)).scalars().all()
out: dict[str, dict[str, float]] = {}
for r in await latest_values(session, SOURCE_MODULE, host_name, include_children=True):
for r in rows:
out.setdefault(r.resource_name, {})[r.metric_name] = r.value
return out
plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py
-24
View File
@@ -1,24 +0,0 @@
# plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py
"""host_agent: how far each host's clock is from Steward's (milestone 490)
Revision ID: host_agent_003_clock_offset
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "host_agent_003_clock_offset"
down_revision: Union[str, None] = "host_agent_002_host_ip"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("host_agent_registrations", sa.Column("clock_offset_s", sa.Float(), nullable=True))
op.add_column("host_agent_registrations",
sa.Column("clock_checked_at", sa.DateTime(timezone=True), nullable=True))
def downgrade() -> None:
op.drop_column("host_agent_registrations", "clock_checked_at")
op.drop_column("host_agent_registrations", "clock_offset_s")
plugins/host_agent/models.py
+1 -6
View File
@@ -2,7 +2,7 @@
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from sqlalchemy import Column, String, DateTime, Float, ForeignKey
from sqlalchemy import Column, String, DateTime, ForeignKey
from steward.models.base import Base
@@ -29,11 +29,6 @@ class HostAgentRegistration(Base):
# Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix).
host_ip = Column(String(45), nullable=True)
last_seen_at = Column(DateTime(timezone=True), nullable=True)
# How far this host's clock is from Steward's, in seconds, positive when
# the host is ahead, measured from the agent's sent_at on the newest post;
# and when that was. None until an agent that stamps its posts reports.
clock_offset_s = Column(Float, nullable=True)
clock_checked_at = Column(DateTime(timezone=True), nullable=True)
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
updated_at = Column(DateTime(timezone=True), nullable=False,
default=_utcnow, onupdate=_utcnow)
plugins/host_agent/routes.py
+263 -293
View File
@@ -1,9 +1,7 @@
# plugins/host_agent/routes.py
from __future__ import annotations
import asyncio
import hashlib
import time
from datetime import datetime, timezone
from typing import Any
@@ -11,13 +9,11 @@ from pathlib import Path
import secrets
from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for, session
from steward.auth.middleware import public_route, require_role
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from sqlalchemy import select, or_, and_
from sqlalchemy import select, func, or_, and_
from datetime import timedelta
from steward.core.formatting import worst_reading
from steward.core.metric_series import points as metric_points
from steward.core.settings import get_setting, public_base_url
from steward.core.time_range import parse_range, RANGE_OPTIONS
from steward.models.hosts import Host
@@ -44,35 +40,6 @@ def _parse_ts(ts: str) -> datetime:
return datetime.fromisoformat(ts)
def clock_offset(sent_at, received_at: datetime) -> float | None:
"""Seconds this host's clock is ahead of Steward's (negative: behind),
from the agent's sent_at and the moment Steward received the post. It
carries the request's own latency, well under a second on a LAN. None
when the post has no usable sent_at (an agent older than 1.12.0)."""
if not isinstance(sent_at, str):
return None
try:
sent = _parse_ts(sent_at)
except ValueError:
return None
if sent.tzinfo is None:
return None
return round((sent - received_at).total_seconds(), 2)
def clock_words(offset: float | None, thresholds: dict) -> tuple[str, str]:
"""('clock 0.3s ahead', 'ok') or ('clock 2m 10s behind', 'warn'): how far
a host's clock is from Steward's, graded by Settings → Thresholds' clock
drift, either way."""
from steward.core.formatting import human_duration, status_class
if offset is None:
return "", "unknown"
off = abs(offset)
words = "clock in step" if off < 1 else \
f"clock {human_duration(off)} {'ahead' if offset > 0 else 'behind'}"
return words, status_class(off, "clock", thresholds)
def pick_host_address(current: str | None, reported: str | None) -> str | None:
"""Return the address to store on the Host, or None for no change.
@@ -199,10 +166,7 @@ def _error(status: int, code: str, detail: str | None = None) -> tuple[Any, int]
@host_agent_bp.post("/ingest")
@public_route("the agent authenticates with its bearer token")
async def ingest():
# Taken first, so the clock offset carries as little of this handler as it can.
received_at = datetime.now(timezone.utc)
auth = request.headers.get("Authorization", "")
if not auth.startswith("Bearer "):
return _error(401, "invalid_token")
@@ -222,214 +186,110 @@ async def ingest():
return _error(400, "malformed_payload", "'samples' must be a non-empty list")
token_hash = _hash_token(raw)
phases = _Phases()
samples, dropped = newest_samples(samples, MAX_SAMPLES_PER_INGEST)
if dropped:
current_app.logger.warning(
"host_agent ingest: kept the newest %d of %d samples for token=%s…",
len(samples), len(samples) + dropped, token_hash[:8])
try:
async with current_app.db_sessionmaker() as session:
reg = (await session.execute(
select(HostAgentRegistration).where(
HostAgentRegistration.token_hash == token_hash)
)).scalar_one_or_none()
if reg is None:
return _error(401, "invalid_token")
host = (await session.execute(
select(Host).where(Host.id == reg.host_id)
)).scalar_one_or_none()
if host is None:
return _error(401, "invalid_token")
async with current_app.db_sessionmaker() as session:
reg = (await session.execute(
select(HostAgentRegistration).where(
HostAgentRegistration.token_hash == token_hash)
)).scalar_one_or_none()
if reg is None:
return _error(401, "invalid_token")
host = (await session.execute(
select(Host).where(Host.id == reg.host_id)
)).scalar_one_or_none()
if host is None:
return _error(401, "invalid_token")
accepted = 0
metric_rows: list[PluginMetric] = []
latest_ts: datetime | None = None
docker_snapshots: list[tuple[datetime, list]] = []
docker_log_batches: list[tuple[datetime, list]] = []
latest_swarm: dict | None = None
latest_swarm_ts: datetime | None = None
latest_disk: dict | None = None
latest_disk_ts: datetime | None = None
# Whether Docker answered, from the newest sample that says (agent ≥ 1.9.0).
docker_status: dict | None = None
for sample in samples:
accepted = 0
latest_ts: datetime | None = None
docker_snapshots: list[tuple[datetime, list]] = []
latest_swarm: dict | None = None
latest_swarm_ts: datetime | None = None
latest_disk: dict | None = None
latest_disk_ts: datetime | None = None
for sample in samples:
try:
recorded_at = _parse_ts(sample["ts"])
except (KeyError, ValueError):
continue
metrics = _expand_sample_to_metrics(sample, host.name, recorded_at)
for m in metrics:
session.add(m)
docker = sample.get("docker")
if isinstance(docker, list) and docker:
docker_snapshots.append((recorded_at, docker))
# Swarm is current-state, not time-series — keep only the newest
# sample's topology (a manager re-reports it every interval).
swarm = sample.get("swarm")
if isinstance(swarm, dict) and (
latest_swarm_ts is None or recorded_at > latest_swarm_ts):
latest_swarm, latest_swarm_ts = swarm, recorded_at
# Disk usage is current-state too — keep only the newest sample's.
disk = sample.get("docker_disk")
if isinstance(disk, dict) and (
latest_disk_ts is None or recorded_at > latest_disk_ts):
latest_disk, latest_disk_ts = disk, recorded_at
accepted += 1
if latest_ts is None or recorded_at > latest_ts:
latest_ts = recorded_at
if accepted == 0:
await session.rollback()
return _error(400, "malformed_payload", "no valid samples")
# Hand host-scoped container data to the docker plugin if it's enabled
# (opportunistic synergy via the capability registry — no hard import,
# no-op when docker is disabled). A failure here must never sink the
# whole ingest, so the metrics above still land.
if docker_snapshots or latest_swarm is not None or latest_disk is not None:
from steward.core.capabilities import has_capability, invoke_capability
if has_capability("docker.persist_host_samples"):
try:
recorded_at = _parse_ts(sample["ts"])
except (KeyError, ValueError):
continue
metric_rows.extend(_expand_sample_to_metrics(sample, host.name, recorded_at))
# An empty list is a real answer: Docker is up with no containers.
docker = sample.get("docker")
if isinstance(docker, list):
docker_snapshots.append((recorded_at, docker))
status = sample.get("docker_status")
if status in ("ok", "unavailable", "error") and (
docker_status is None or recorded_at > docker_status["at"]):
docker_status = {
"status": status, "detail": sample.get("docker_detail"),
"at": recorded_at, "swarm_manager": sample.get("swarm_manager")}
# Container logs are time-series (append every line, not newest-only);
# each record carries its own Docker ts, recorded_at is the fallback.
docker_logs = sample.get("docker_logs")
if isinstance(docker_logs, list) and docker_logs:
docker_log_batches.append((recorded_at, docker_logs))
# Swarm is current-state, not time-series — keep only the newest
# sample's topology (a manager re-reports it every interval).
swarm = sample.get("swarm")
if isinstance(swarm, dict) and (
latest_swarm_ts is None or recorded_at > latest_swarm_ts):
latest_swarm, latest_swarm_ts = swarm, recorded_at
# Disk usage is current-state too — keep only the newest sample's.
disk = sample.get("docker_disk")
if isinstance(disk, dict) and (
latest_disk_ts is None or recorded_at > latest_disk_ts):
latest_disk, latest_disk_ts = disk, recorded_at
accepted += 1
if latest_ts is None or recorded_at > latest_ts:
latest_ts = recorded_at
# SAVEPOINT so a docker-side failure rolls back only the
# docker writes, leaving the host metrics intact to commit.
async with session.begin_nested():
await invoke_capability(
"docker.persist_host_samples", UserRole.admin,
session, host, docker_snapshots, latest_swarm,
latest_disk,
)
except Exception:
current_app.logger.exception(
"host_agent ingest: docker persist failed for host=%s",
host.name)
if accepted == 0:
await session.rollback()
return _error(400, "malformed_payload", "no valid samples")
# How far this host's clock is from Steward's, either way, so an
# alert rule reads "> 30" (milestone 490).
offset = clock_offset(payload.get("sent_at"), received_at)
if offset is not None:
metric_rows.append(PluginMetric(
source_module=SOURCE_MODULE, resource_name=host.name,
metric_name="clock_off_secs", value=abs(offset), recorded_at=received_at))
phases.mark("parse")
# Through the alert pipeline, so rules on host metrics can fire.
from steward.core.alerts import record_metrics
phases.size = f"samples={accepted} metric_rows={len(metric_rows)}"
await record_metrics(session, metric_rows, phases)
# Hand host-scoped container data to the docker plugin if it's enabled
# (opportunistic synergy via the capability registry — no hard import,
# no-op when docker is disabled). A failure here must never sink the
# whole ingest, so the metrics above still land.
if (docker_snapshots or latest_swarm is not None or docker_status is not None
or latest_disk is not None or docker_log_batches):
from steward.core.capabilities import has_capability, invoke_capability
if has_capability("docker.persist_host_samples"):
try:
# SAVEPOINT so a docker-side failure rolls back only the
# docker writes, leaving the host metrics intact to commit.
async with session.begin_nested():
await invoke_capability(
"docker.persist_host_samples", UserRole.admin,
session, host, docker_snapshots, latest_swarm,
latest_disk, docker_log_batches, docker_status,
phases=phases,
)
except Exception:
current_app.logger.exception(
"host_agent ingest: docker persist failed for host=%s",
host.name)
phases.mark("docker")
md = payload.get("metadata") or {}
changed = False
if latest_ts and (reg.last_seen_at is None or latest_ts > reg.last_seen_at):
reg.last_seen_at = latest_ts
md = payload.get("metadata") or {}
changed = False
if latest_ts and (reg.last_seen_at is None or latest_ts > reg.last_seen_at):
reg.last_seen_at = latest_ts
changed = True
version = payload.get("agent_version")
if version and reg.agent_version != version:
reg.agent_version = version
changed = True
for field in ("kernel", "distro", "arch"):
if field in md and getattr(reg, field) != md[field]:
setattr(reg, field, md[field])
changed = True
version = payload.get("agent_version")
if version and reg.agent_version != version:
reg.agent_version = version
changed = True
for field in ("kernel", "distro", "arch"):
if field in md and getattr(reg, field) != md[field]:
setattr(reg, field, md[field])
changed = True
host_ip = md.get("host_ip")
if host_ip and reg.host_ip != host_ip:
reg.host_ip = host_ip
changed = True
# Mirror the reported IP into Host.address only when it's blank.
new_address = pick_host_address(host.address, host_ip)
if new_address is not None:
host.address = new_address
if changed:
reg.updated_at = datetime.now(timezone.utc)
host_ip = md.get("host_ip")
if host_ip and reg.host_ip != host_ip:
reg.host_ip = host_ip
changed = True
# Mirror the reported IP into Host.address only when it's blank.
new_address = pick_host_address(host.address, host_ip)
if new_address is not None:
host.address = new_address
if changed:
reg.updated_at = datetime.now(timezone.utc)
if offset is not None:
reg.clock_offset_s, reg.clock_checked_at = offset, received_at
clock_th = current_app.config.get("THRESHOLDS", {}).get("clock") or {"warn": 30}
if abs(offset) > clock_th["warn"]:
current_app.logger.warning(
"host_agent ingest: host=%s clock is %.1fs %s Steward's",
host.name, abs(offset), "ahead of" if offset > 0 else "behind")
if latest_ts:
skew = abs((datetime.now(timezone.utc) - latest_ts).total_seconds())
if skew > 300:
current_app.logger.warning(
"host_agent ingest: clock skew %.0fs for host=%s", skew, host.name)
await session.commit()
phases.mark("commit")
except asyncio.CancelledError:
# The agent hung up (its 10 s timeout) and the server dropped the
# request: say how far it got before re-raising.
phases.cancelled(token_hash[:8])
raise
await session.commit()
phases.report(host.name, accepted, len(metric_rows),
sum(len(c) for _, c in docker_snapshots),
sum(len(lines) for _, lines in docker_log_batches))
return jsonify({"ok": True, "samples_accepted": accepted, "samples_dropped": dropped}), 200
# The agent gives up on a post after 10 s (agent.py); an ingest past this
# says where its time went, so a slow phase shows in the logs (#5149).
SLOW_INGEST_SECONDS = 2.0
# How many samples one ingest writes (#5149). An agent that could not post
# sends its whole buffer (up to 21 samples, ~4,000 metric rows); on the live
# instance that did not finish inside the agent's 10 s, the rollback left the
# buffer full, and every later post failed the same way. The newest samples
# are kept: the older ones would be lost to the agent's ring buffer anyway.
# Five matches the agent's MAX_SAMPLES_PER_POST (1.10+), so its backlog posts
# land whole: ~1,000 metric rows at the 1,300-4,000 rows/s measured once
# plugin_metrics got time-ordered ids, with posts answering in 0.4-1.3 s (#5181).
MAX_SAMPLES_PER_INGEST = 5
def newest_samples(samples: list, limit: int) -> tuple[list, int]:
"""(the newest `limit` samples, oldest first; how many were left out).
Ordered by their ISO-8601 `ts` (the agent writes them all in one form);
one without a ts sorts oldest and is rejected later as malformed."""
if len(samples) <= limit:
return samples, 0
ordered = sorted(samples, key=lambda s: str(s.get("ts") or "") if isinstance(s, dict) else "")
return ordered[-limit:], len(samples) - limit
class _Phases:
"""Wall-clock per phase of one ingest request."""
def __init__(self):
self.start = self.last = time.monotonic()
self.took: dict[str, float] = {}
self.size = ""
def mark(self, name: str) -> None:
now = time.monotonic()
self.took[name] = self.took.get(name, 0.0) + (now - self.last)
self.last = now
def cancelled(self, who: str) -> None:
current_app.logger.warning(
"host_agent ingest: dropped after %.1fs for token=%s… (completed: %s) %s",
time.monotonic() - self.start, who,
" ".join(f"{k}={v:.1f}s" for k, v in self.took.items()) or "nothing", self.size)
def report(self, host: str, samples: int, metrics: int, containers: int, log_lines: int) -> None:
total = time.monotonic() - self.start
if total < SLOW_INGEST_SECONDS:
return
current_app.logger.warning(
"host_agent ingest: slow %.1fs for host=%s (%s) samples=%d metric_rows=%d "
"containers=%d log_lines=%d", total, host,
" ".join(f"{k}={v:.1f}s" for k, v in self.took.items()),
samples, metrics, containers, log_lines)
return jsonify({"ok": True, "samples_accepted": accepted}), 200
AGENT_SOURCE_PATH = Path(__file__).parent / "agent.py"
@@ -446,7 +306,6 @@ def _agent_version() -> str:
@host_agent_bp.get("/install.sh")
@public_route("fetched by curl on the host being enrolled; carries an enrolment token")
async def install_script():
token = request.args.get("token", "")
if not token:
@@ -478,7 +337,6 @@ async def install_script():
@host_agent_bp.get("/agent.py")
@public_route("fetched by the installer and by agent self-update; holds no secrets")
async def agent_source():
try:
body = AGENT_SOURCE_PATH.read_text(encoding="utf-8")
@@ -487,28 +345,6 @@ async def agent_source():
return Response(body, mimetype="text/x-python")
_FLEET_ORDER = {"stale": 0, "pending": 1, "up": 2}
def fleet_state(reg, now: datetime, stale_after: int) -> str:
"""An agent as the fleet table shows it: "up" while it reports, "stale"
once its last report is older than stale_after, "pending" if it never has."""
if reg.last_seen_at is None:
return "pending"
return "stale" if (now - reg.last_seen_at).total_seconds() > stale_after else "up"
def fleet_rows(regs, hosts_by_id: dict, now: datetime, stale_after: int) -> list[dict]:
"""The fleet table: silent agents first, then never-reported, then live,
each by host name."""
rows = []
for r in regs:
host = hosts_by_id.get(r.host_id)
rows.append({"reg": r, "host": host, "name": host.name if host else r.host_id,
"state": fleet_state(r, now, stale_after)})
return sorted(rows, key=lambda x: (_FLEET_ORDER[x["state"]], x["name"].lower()))
def _stale_after_seconds() -> int:
return int(
current_app.config.get("PLUGINS", {})
@@ -517,8 +353,124 @@ def _stale_after_seconds() -> int:
)
async def _fleet_rows(session) -> list[dict]:
"""One row per registered host with its latest host-level metrics + stale flag.
Only bare host-level resources are used here (sub-resources carry a ':' and
are skipped) so the fleet glance stays one line per host.
"""
regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
host_ids = [r.host_id for r in regs]
hosts = {
h.id: h for h in (await session.execute(
select(Host).where(Host.id.in_(host_ids))
)).scalars().all()
} if host_ids else {}
subq = (
select(
PluginMetric.resource_name,
PluginMetric.metric_name,
func.max(PluginMetric.recorded_at).label("max_ts"),
)
.where(PluginMetric.source_module == SOURCE_MODULE)
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
).subquery()
latest_rows = (await session.execute(
select(PluginMetric).join(
subq,
(PluginMetric.resource_name == subq.c.resource_name) &
(PluginMetric.metric_name == subq.c.metric_name) &
(PluginMetric.recorded_at == subq.c.max_ts),
).where(PluginMetric.source_module == SOURCE_MODULE)
)).scalars().all()
latest: dict[str, dict[str, float]] = {}
root_disk: dict[str, float] = {} # host_name → root (/) disk %
for row in latest_rows:
if row.resource_name.endswith(":/") and row.metric_name == "disk_used_pct":
root_disk[row.resource_name[:-2]] = row.value
if ":" in row.resource_name:
continue
latest.setdefault(row.resource_name, {})[row.metric_name] = row.value
# Per-host recent series for the inline sparklines (cpu/mem/load host-level,
# disk from the root mount), so each metric shows its trend beside the value —
# the host-panel at-a-glance, on the fleet widget. 1h window keeps it cheap.
host_names = [h.name for h in hosts.values()]
spark_series: dict[str, dict[str, list]] = {
n: {"cpu": [], "mem": [], "disk": [], "load": []} for n in host_names
}
if host_names:
since = datetime.now(timezone.utc) - timedelta(hours=1)
sp_rows = (await session.execute(
select(PluginMetric).where(
PluginMetric.source_module == SOURCE_MODULE,
PluginMetric.recorded_at >= since,
or_(
and_(PluginMetric.resource_name.in_(host_names),
PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))),
and_(PluginMetric.resource_name.in_([n + ":/" for n in host_names]),
PluginMetric.metric_name == "disk_used_pct"),
),
).order_by(PluginMetric.recorded_at)
)).scalars().all()
for r in sp_rows:
if r.resource_name.endswith(":/"):
name = r.resource_name[:-2]
if name in spark_series:
spark_series[name]["disk"].append(r.value)
else:
key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name)
if key and r.resource_name in spark_series:
spark_series[r.resource_name][key].append(r.value)
from steward.core.status import sparkline_svg
stale_after = _stale_after_seconds()
now = datetime.now(timezone.utc)
rows = []
for reg in regs:
host = hosts.get(reg.host_id)
if host is None:
continue
m = latest.get(host.name, {})
ls = reg.last_seen_at
stale = ls is None or (now - ls).total_seconds() > stale_after
ser = spark_series.get(host.name, {})
sparks = {
k: (sparkline_svg(v[-60:], width=72, height=16) if len(v) >= 2 else "")
for k, v in ser.items()
}
rows.append({
"host": host,
"reg": reg,
"stale": stale,
"cpu_pct": m.get("cpu_pct"),
"mem_used_pct": m.get("mem_used_pct"),
"disk_worst": m.get("disk_used_pct_worst"), # health dot only
"disk_root": root_disk.get(host.name), # displayed at-a-glance
"load_1m": m.get("load_1m"),
"temp_max": m.get("temp_c_max"),
"net_rx_bps": m.get("net_rx_bps"),
"net_tx_bps": m.get("net_tx_bps"),
"disk_read_bps": m.get("disk_read_bps"),
"disk_write_bps": m.get("disk_write_bps"),
"sparks": sparks,
})
rows.sort(key=lambda r: r["host"].name.lower())
return rows
@host_agent_bp.get("/widget")
async def widget_table():
"""Fleet-glance dashboard widget: one row per monitored host, latest metrics."""
async with current_app.db_sessionmaker() as session:
rows = await _fleet_rows(session)
return await render_template("widget_table.html", rows=rows)
@host_agent_bp.get("/widget/history")
@require_role(UserRole.viewer)
async def widget_history():
host_id = request.args.get("host_id", "")
try:
@@ -712,25 +664,37 @@ async def host_vitals(host_id: str):
HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
latest = await _latest_metrics_for_host(db, host.name) if reg else {}
# The last 6 h for the at-a-glance sparklines (cpu/mem/load host-level,
# disk from the root mount), averaged into ~120 points in SQL, so the
# line spans the whole window and only what is drawn is fetched.
# Recent series for the at-a-glance sparklines (cpu/mem/load host-level,
# disk from the root mount sub-resource).
series_raw: dict[str, list[float]] = {"cpu": [], "mem": [], "disk": [], "load": []}
if reg:
since = datetime.now(timezone.utc) - timedelta(hours=6)
for key, resource, metric in (("cpu", host.name, "cpu_pct"),
("mem", host.name, "mem_used_pct"),
("load", host.name, "load_1m"),
("disk", host.name + ":/", "disk_used_pct")):
series_raw[key] = [v for _t, v in await metric_points(
db, SOURCE_MODULE, resource, metric, since, target=120)]
spark_rows = (await db.execute(
select(PluginMetric).where(
PluginMetric.source_module == SOURCE_MODULE,
PluginMetric.recorded_at >= since,
or_(
and_(PluginMetric.resource_name == host.name,
PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))),
and_(PluginMetric.resource_name == host.name + ":/",
PluginMetric.metric_name == "disk_used_pct"),
),
).order_by(PluginMetric.recorded_at)
)).scalars().all()
for r in spark_rows:
if r.resource_name == host.name:
key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name)
if key:
series_raw[key].append(r.value)
else:
series_raw["disk"].append(r.value)
from steward.core.status import sparkline_svg
hostlvl = latest.get(host.name, {})
# At-a-glance disk = the root filesystem (what people actually care about),
# not the "worst" mount which is opaque. Worst stays on the full-metrics page.
disk_root = (latest.get(host.name + ":/", {}) or {}).get("disk_used_pct")
sparks = {k: sparkline_svg(v, height=24, fill_width=True) for k, v in series_raw.items()}
sparks = {k: sparkline_svg(v[-120:]) for k, v in series_raw.items()}
# Normalize load by core count so it's comparable across hardware (load/core
# as %). Cores derived from the per-core CPU sub-resources already collected.
cores = sum(1 for k in latest if k.startswith(host.name + ":core"))
@@ -744,15 +708,11 @@ async def host_vitals(host_id: str):
ls = reg.last_seen_at if reg else None
reporting = bool(reg) and ls is not None
stale = reporting and (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
cpu, mem = hostlvl.get("cpu_pct"), hostlvl.get("mem_used_pct")
worst = worst_reading((("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("Disk /", disk_root, "disk"),
("Load per core", load_per_core, "load")),
current_app.config.get("THRESHOLDS", {}))
return await render_template(
"_host_vitals.html",
reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core,
sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst,
clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})),
reg=reg, cpu=hostlvl.get("cpu_pct"), mem=hostlvl.get("mem_used_pct"),
disk_root=disk_root, load_per_core=load_per_core, sparks=sparks, psi=psi,
reporting=reporting, stale=stale,
)
@@ -824,10 +784,11 @@ async def fleet():
managed_key_set = bool(
(current_app.config.get("ANSIBLE", {}).get("ssh_public_key") or "").strip())
now, stale_after = datetime.now(timezone.utc), _stale_after_seconds()
return await render_template(
"settings_list.html",
registrations=fleet_rows(regs, hosts_by_id, now, stale_after),
registrations=[
{"reg": r, "host": hosts_by_id.get(r.host_id)} for r in regs
],
new_token=new_token,
install_url=install_url,
ansible_available=ansible_available,
@@ -837,24 +798,35 @@ async def fleet():
)
@host_agent_bp.post("/fleet/<host_id>/register")
@host_agent_bp.post("/fleet/add-host")
@require_role(UserRole.admin)
async def register_host(host_id: str):
"""Issue an agent token for an existing host, for a manual install: the
fleet page then shows the one-line install command, once. Hosts are added
on Hosts → Add host; this is only the agent half (#5282)."""
async def add_host():
form = await request.form
name = (form.get("name") or "").strip()
address = (form.get("address") or "").strip()
if not name:
return _error(400, "missing_name")
async with current_app.db_sessionmaker() as session:
host = await session.get(Host, host_id)
host = (await session.execute(
select(Host).where(Host.name == name))).scalar_one_or_none()
if host is None:
return _error(404, "not_found")
host = Host(name=name, address=address)
session.add(host)
await session.flush()
existing = (await session.execute(
select(HostAgentRegistration).where(
HostAgentRegistration.host_id == host.id))).scalar_one_or_none()
if existing is not None:
await session.rollback()
return _error(400, "already_registered")
raw, hashed = _new_token_pair()
session.add(HostAgentRegistration(host_id=host.id, token_hash=hashed))
reg = HostAgentRegistration(host_id=host.id, token_hash=hashed)
session.add(reg)
await session.commit()
host_id = host.id
return redirect(url_for("host_agent.fleet") + f"?new_token={raw}&host_id={host_id}")
@@ -975,7 +947,6 @@ async def deploy_via_ansible():
playbook_path="host_agent/install.yml",
inventory_content=inventory_content,
inventory_scope=scope,
redact=list(tokens.values()),
# extra-vars outrank the playbook's `vars:` defaults; token stays per-host.
params={"extra_vars_map": {"steward_url": url, "agent_interval": interval}},
triggered_by=session.get("user_id"),
@@ -1053,7 +1024,6 @@ async def provision_via_ansible():
playbook_path="host_agent/provision.yml",
inventory_content=inventory_content,
inventory_scope=scope,
redact=list(tokens.values()),
# extra_vars_map → a JSON -e @file: highest precedence (beats play vars)
# and space-safe (steward_pubkey carries a comment with a space).
params={"extra_vars_map": {
plugins/host_agent/scheduler.py
+51 -38
View File
@@ -1,12 +1,11 @@
# plugins/host_agent/scheduler.py
from __future__ import annotations
import logging
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from typing import Iterable
from sqlalchemy import select
from steward.core.alerts import MetricBatch
from steward.core.scheduler import ScheduledTask
from steward.models.hosts import Host
from .models import HostAgentRegistration
@@ -14,52 +13,66 @@ from .models import HostAgentRegistration
logger = logging.getLogger(__name__)
def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
now: datetime) -> list[tuple[str, float]]:
"""(host name, seconds since its agent last reported) per registration.
An agent that has never reported counts from when it was registered. A
registration whose host is gone is skipped.
"""
out = []
for r in regs:
name = host_names.get(r.host_id)
seen = r.last_seen_at or r.created_at
if name is None or seen is None:
continue
out.append((name, max(0.0, (now - seen).total_seconds())))
return out
def _filter_stale(
regs: Iterable,
*,
now: datetime,
stale_after_seconds: int,
) -> list:
"""Pure staleness filter: returns the subset with last_seen_at strictly
older than (now - stale_after_seconds). Rows with last_seen_at=None are
never stale (they are unregistered-in-practice)."""
cutoff = now - timedelta(seconds=stale_after_seconds)
return [
r for r in regs
if r.last_seen_at is not None and r.last_seen_at < cutoff
]
async def record_seconds_since_seen(app) -> None:
"""Record host_agent/<host>/seconds_since_seen for every registered agent,
so "agent offline" is an ordinary alert rule (seconds_since_seen > 180)."""
now = datetime.now(timezone.utc)
async def find_stale_registrations(app, stale_after_seconds: int = 180) -> list[dict]:
async with app.db_sessionmaker() as session:
async with session.begin():
regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
if not regs:
return
host_names = dict((await session.execute(
select(Host.id, Host.name).where(Host.id.in_([r.host_id for r in regs]))
)).all())
batch = MetricBatch()
for name, seconds in seconds_since_seen(regs, host_names, now):
batch.add(source_module="host_agent", resource_name=name,
metric_name="seconds_since_seen", value=seconds, recorded_at=now)
await batch.record(session)
all_regs = (await session.execute(
select(HostAgentRegistration)
)).scalars().all()
stale_rows = _filter_stale(
all_regs,
now=datetime.now(timezone.utc),
stale_after_seconds=stale_after_seconds,
)
if not stale_rows:
return []
hosts = {
h.id: h for h in (await session.execute(
select(Host).where(Host.id.in_([r.host_id for r in stale_rows]))
)).scalars().all()
}
return [
{
"host_id": r.host_id,
"host_name": hosts[r.host_id].name if r.host_id in hosts else "?",
"last_seen_at": r.last_seen_at,
}
for r in stale_rows
]
def make_task(app) -> ScheduledTask:
async def _record():
async def _check_stale():
try:
await record_seconds_since_seen(app)
stale = await find_stale_registrations(app)
except Exception:
logger.exception("host_agent: recording seconds_since_seen failed")
logger.exception("host_agent stale check failed")
return
if stale:
logger.info(
"host_agent: %d stale agent(s): %s",
len(stale),
[s["host_name"] for s in stale],
)
return ScheduledTask(
name="host_agent_seconds_since_seen",
coro_factory=_record,
name="host_agent_stale_check",
coro_factory=_check_stale,
interval_seconds=60,
run_on_startup=False,
)
plugins/host_agent/templates/_host_metrics.html
+42 -25
View File
@@ -1,15 +1,32 @@
{# Current-state fragment for the full-metrics page — polled live via HTMX.
Each poll re-renders the latest snapshot the server has. #}
{% from "_macros.html" import status_badge %}
{# A percentage bar graded by Settings → Thresholds for `kind` (cpu, mem, disk). #}
{% macro bar(pct, kind) %}
<div class="meter"><div class="meter-fill meter-{{ status_class(pct, kind) }}" style="width:{{ (pct or 0)|round(1) }}%;"></div></div>
Self-contained (defines its own format macros) so each poll re-renders the
latest snapshot the server has. #}
{% macro fmt_bps(v) %}
{%- if v is none -%}—
{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s
{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s
{%- else -%}{{ "%.0f"|format(v) }} B/s
{%- endif -%}
{% endmacro %}
{% macro fmt_bytes(v) %}
{%- if v is none -%}—
{%- elif v >= 1125899906842624 -%}{{ "%.1f"|format(v / 1125899906842624) }} PB
{%- elif v >= 1099511627776 -%}{{ "%.1f"|format(v / 1099511627776) }} TB
{%- elif v >= 1073741824 -%}{{ "%.1f"|format(v / 1073741824) }} GB
{%- elif v >= 1048576 -%}{{ "%.0f"|format(v / 1048576) }} MB
{%- else -%}{{ "%.0f"|format(v / 1024) }} KB
{%- endif -%}
{% endmacro %}
{% macro bar(pct) %}
{%- set p = pct if pct is not none else 0 -%}
<div style="height:8px;background:var(--bg-elevated);border-radius:4px;overflow:hidden;border:1px solid var(--border);">
<div style="height:100%;width:{{ p|round(1) }}%;background:{% if pct is none %}var(--text-dim){% elif pct < 70 %}var(--green){% elif pct < 90 %}var(--yellow){% else %}var(--red){% endif %};"></div>
</div>
{% endmacro %}
{% macro gauge(value, kind, text) %}<span class="stat-val tone-{{ status_class(value, kind) }}" style="font-size:1.6rem;">{{ text }}</span>{% endmacro %}
{# ── Identity / metadata ─────────────────────────────────────────────────── #}
<div class="card" style="display:flex;flex-wrap:wrap;gap:0.5rem 1.25rem;align-items:center;font-size:0.82rem;color:var(--text-muted);">
<span>{{ status_badge('stale' if stale else 'up', 'Stale' if stale else 'Live') }}</span>
<span>{% if stale %}<span class="badge badge-red">stale</span>{% else %}<span class="badge badge-green">live</span>{% endif %}</span>
<span><span style="color:var(--text-dim);">Address</span> {{ host.address or "—" }}</span>
{% if reg %}
{% if reg.distro %}<span><span style="color:var(--text-dim);">OS</span> {{ reg.distro }}</span>{% endif %}
@@ -17,8 +34,8 @@
{% if reg.arch %}<span><span style="color:var(--text-dim);">Arch</span> {{ reg.arch }}</span>{% endif %}
{% if reg.agent_version %}<span><span style="color:var(--text-dim);">Agent</span> v{{ reg.agent_version }}</span>{% endif %}
{% set up = hostlvl.get('uptime_secs') %}
{% if up %}<span><span style="color:var(--text-dim);">Uptime</span> {{ up|duration }}</span>{% endif %}
<span><span style="color:var(--text-dim);">Last seen</span> {{ reg.last_seen_at|ago if reg.last_seen_at else "never" }}</span>
{% if up %}<span><span style="color:var(--text-dim);">Uptime</span> {{ (up // 86400)|int }}d {{ ((up % 86400) // 3600)|int }}h</span>{% endif %}
<span><span style="color:var(--text-dim);">Last seen</span> {{ reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") ~ " UTC" if reg.last_seen_at else "never" }}</span>
{% else %}
<span>No agent registration found for this host.</span>
{% endif %}
@@ -33,16 +50,16 @@
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">CPU</div>
{% set c = hostlvl.get('cpu_pct') %}
{{ gauge(c, 'cpu', ("%.0f%%"|format(c)) if c is not none else "—") }}
<div style="margin-top:0.5rem;">{{ bar(c, 'cpu') }}</div>
<span class="stat-val" style="font-size:1.6rem;color:{% if c is none %}var(--text-dim){% elif c < 70 %}var(--green){% elif c < 90 %}var(--yellow){% else %}var(--red){% endif %};">{% if c is not none %}{{ "%.0f"|format(c) }}%{% else %}—{% endif %}</span>
<div style="margin-top:0.5rem;">{{ bar(c) }}</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Memory</div>
{% set mp = hostlvl.get('mem_used_pct') %}
{{ gauge(mp, 'mem', ("%.0f%%"|format(mp)) if mp is not none else "—") }}
<div style="margin-top:0.5rem;">{{ bar(mp, 'mem') }}</div>
<span class="stat-val" style="font-size:1.6rem;color:{% if mp is none %}var(--text-dim){% elif mp < 70 %}var(--green){% elif mp < 90 %}var(--yellow){% else %}var(--red){% endif %};">{% if mp is not none %}{{ "%.0f"|format(mp) }}%{% else %}—{% endif %}</span>
<div style="margin-top:0.5rem;">{{ bar(mp) }}</div>
<div style="margin-top:0.45rem;font-size:0.72rem;color:var(--text-dim);">
avail {{ hostlvl.get('mem_available_bytes')|bytes }} · cache {{ hostlvl.get('mem_cached_bytes')|bytes }} · swap {{ hostlvl.get('swap_used_bytes')|bytes }}
avail {{ fmt_bytes(hostlvl.get('mem_available_bytes')) }} · cache {{ fmt_bytes(hostlvl.get('mem_cached_bytes')) }} · swap {{ fmt_bytes(hostlvl.get('swap_used_bytes')) }}
</div>
</div>
<div class="card" style="margin-bottom:0;">
@@ -56,22 +73,22 @@
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Network</div>
<div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
<div><span style="color:var(--green);">↓</span> {{ hostlvl.get('net_rx_bps')|rate }}</div>
<div><span style="color:var(--red);">↑</span> {{ hostlvl.get('net_tx_bps')|rate }}</div>
<div><span style="color:var(--green);">↓</span> {{ fmt_bps(hostlvl.get('net_rx_bps')) }}</div>
<div><span style="color:var(--red);">↑</span> {{ fmt_bps(hostlvl.get('net_tx_bps')) }}</div>
</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Disk I/O</div>
<div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
<div><span style="color:var(--text-dim);">rd</span> {{ hostlvl.get('disk_read_bps')|rate }}</div>
<div><span style="color:var(--text-dim);">wr</span> {{ hostlvl.get('disk_write_bps')|rate }}</div>
<div><span style="color:var(--text-dim);">rd</span> {{ fmt_bps(hostlvl.get('disk_read_bps')) }}</div>
<div><span style="color:var(--text-dim);">wr</span> {{ fmt_bps(hostlvl.get('disk_write_bps')) }}</div>
</div>
</div>
{% if hostlvl.get('temp_c_max') is not none %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Temp (max)</div>
{% set t = hostlvl.get('temp_c_max') %}
{{ gauge(t, 'temp', "%.0f°C"|format(t)) }}
<span class="stat-val" style="font-size:1.6rem;color:{% if t < 70 %}var(--green){% elif t < 85 %}var(--yellow){% else %}var(--red){% endif %};">{{ "%.0f"|format(t) }}°C</span>
</div>
{% endif %}
{% if hostlvl.get('psi_mem_some_avg10') is not none %}
@@ -96,7 +113,7 @@
<div style="display:flex;justify-content:space-between;font-size:0.72rem;color:var(--text-dim);margin-bottom:0.2rem;">
<span>core {{ idx }}</span><span>{% if pct is not none %}{{ "%.0f"|format(pct) }}%{% else %}—{% endif %}</span>
</div>
{{ bar(pct, 'cpu') }}
{{ bar(pct) }}
</div>
{% endfor %}
</div>
@@ -111,9 +128,9 @@
<div style="margin-bottom:0.6rem;">
<div style="display:flex;justify-content:space-between;font-size:0.8rem;margin-bottom:0.2rem;">
<span style="font-family:ui-monospace,monospace;">{{ mount }}</span>
<span style="color:var(--text-muted);">{{ m.get('disk_used_bytes')|bytes }} / {{ m.get('disk_total_bytes')|bytes }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %}</span>
<span style="color:var(--text-muted);">{{ fmt_bytes(m.get('disk_used_bytes')) }} / {{ fmt_bytes(m.get('disk_total_bytes')) }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %}</span>
</div>
{{ bar(m.get('disk_used_pct'), 'disk') }}
{{ bar(m.get('disk_used_pct')) }}
</div>
{% endfor %}
</div>
@@ -128,7 +145,7 @@
{% for iface, m in nets.items() %}
<div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
<span style="font-family:ui-monospace,monospace;">{{ iface }}</span>
<span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">↓ {{ m.get('net_rx_bps')|rate }} · ↑ {{ m.get('net_tx_bps')|rate }}</span>
<span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">↓ {{ fmt_bps(m.get('net_rx_bps')) }} · ↑ {{ fmt_bps(m.get('net_tx_bps')) }}</span>
</div>
{% endfor %}
</div>
@@ -139,7 +156,7 @@
{% for dev, m in disks_io.items() %}
<div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
<span style="font-family:ui-monospace,monospace;">{{ dev }}</span>
<span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">rd {{ m.get('disk_read_bps')|rate }} · wr {{ m.get('disk_write_bps')|rate }}</span>
<span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">rd {{ fmt_bps(m.get('disk_read_bps')) }} · wr {{ fmt_bps(m.get('disk_write_bps')) }}</span>
</div>
{% endfor %}
</div>
@@ -155,7 +172,7 @@
{% for label, c in temps.items() %}
<div style="display:flex;justify-content:space-between;gap:0.5rem;font-size:0.8rem;padding:0.15rem 0;">
<span style="white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ label }}</span>
<span class="tone-{{ status_class(c, 'temp') }}" style="font-variant-numeric:tabular-nums;">{% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %}</span>
<span style="color:{% if c is none %}var(--text-dim){% elif c < 70 %}var(--green){% elif c < 85 %}var(--yellow){% else %}var(--red){% endif %};font-variant-numeric:tabular-nums;">{% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %}</span>
</div>
{% endfor %}
</div>
plugins/host_agent/templates/_host_vitals.html
+30 -29
View File
@@ -1,34 +1,35 @@
{# Host health header — live polled into the host hub: live or stale, the worst
reading against Settings → Thresholds, last seen, then CPU / memory / disk /
load tiles graded by the same cutoffs. Renders nothing until the agent
reports (the Agent panel below then shows provisioning). #}
{% from "_macros.html" import status_badge, stat_tile %}
{% macro pct(v) %}{{ '%.0f%%'|format(v) if v is not none else '—' }}{% endmacro %}
{# Host vitals strip — compact CPU/MEM/DISK/LOAD + pressure, live-polled into the
host hub. Renders nothing until the agent reports (the Agent panel below then
shows provisioning). #}
{% if reporting %}
<section class="card host-health" aria-label="Host health">
<div class="host-health-head">
{{ status_badge('stale' if stale else 'up', 'Stale' if stale else 'Live') }}
<span class="host-health-summary">
{%- if stale %}No report since {{ reg.last_seen_at|ago }}; the readings below are from then.
{%- elif worst %}<span class="tone-{{ worst[2] }}">{{ worst[0] }} {{ pct(worst[1]) }}</span> is past its {{ "critical" if worst[2] == "crit" else "warning" }} threshold.
{%- else %}Every reading is within its thresholds.{% endif -%}
{% set lbl = "font-size:0.68rem;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em;" %}
{% macro vital(label, value, text, kind, spark) %}
<div style="min-width:88px;">
<div style="{{ lbl }}">{{ label }}</div>
<div style="font-size:1.5rem;font-weight:700;line-height:1.1;{{ threshold_style(value, kind) }}">{{ text }}</div>
<div style="margin-top:0.2rem;">{{ spark | safe }}</div>
</div>
{% endmacro %}
<div class="card" style="display:flex;flex-wrap:wrap;align-items:flex-start;gap:1rem 2rem;margin-bottom:1rem;">
{{ vital("CPU", cpu, ('%.0f%%'|format(cpu) if cpu is not none else '—'), 'cpu', sparks.cpu) }}
{{ vital("Memory", mem, ('%.0f%%'|format(mem) if mem is not none else '—'), 'mem', sparks.mem) }}
{{ vital("Disk /", disk_root, ('%.0f%%'|format(disk_root) if disk_root is not none else '—'), 'disk', sparks.disk) }}
{{ vital("Load /core", load_per_core, ('%d%%'|format(load_per_core) if load_per_core is not none else '—'), 'load', sparks.load) }}
<div style="margin-left:auto;display:flex;flex-direction:column;align-items:flex-end;gap:0.3rem;font-size:0.75rem;color:var(--text-muted);text-align:right;">
<span>
{% if stale %}<span class="badge badge-red">stale</span>{% else %}<span class="badge badge-green">live</span>{% endif %}
{% if reg and reg.agent_version %}<span style="margin-left:0.3rem;">v{{ reg.agent_version }}</span>{% endif %}
</span>
<span class="problem-detail">
{%- if not stale %}seen {{ reg.last_seen_at|ago }}{% endif %}
{%- if reg.agent_version %}{{ " · " if not stale }}agent v{{ reg.agent_version }}{% endif -%}
{%- if clock and clock[0] %} · <span{% if clock[1] in ('warn', 'crit') %} class="tone-{{ clock[1] }}" title="Further from Steward's clock than Settings → Thresholds allows: check what keeps this host's time in sync"{% endif %}>{{ clock[0] }}</span>{% endif -%}
{% if psi.cpu is not none or psi.mem is not none or psi.io is not none %}
<span title="Pressure stall — % of the last 10s tasks waited for the resource">
<span style="color:var(--text-dim);text-transform:uppercase;font-size:0.66rem;letter-spacing:0.04em;">Pressure 10s</span>
cpu {{ '%.0f%%'|format(psi.cpu) if psi.cpu is not none else '—' }}
· mem {{ '%.0f%%'|format(psi.mem) if psi.mem is not none else '—' }}
· io {{ '%.0f%%'|format(psi.io) if psi.io is not none else '—' }}
</span>
{% endif %}
{% if reg and reg.last_seen_at %}<span>seen {{ reg.last_seen_at.strftime("%H:%M:%S") }} UTC</span>{% endif %}
</div>
<div class="stat-strip host-health-tiles" style="margin-bottom:0;">
{{ stat_tile("CPU", pct(cpu), state=status_class(cpu, 'cpu'), hint=sparks.cpu|safe) }}
{{ stat_tile("Memory", pct(mem), state=status_class(mem, 'mem'), hint=sparks.mem|safe) }}
{{ stat_tile("Disk /", pct(disk_root), state=status_class(disk_root, 'disk'), hint=sparks.disk|safe) }}
{{ stat_tile("Load per core", pct(load_per_core), state=status_class(load_per_core, 'load'), hint=sparks.load|safe) }}
</div>
{% if psi.cpu is not none or psi.mem is not none or psi.io is not none %}
<p class="field-hint" style="margin-top:0.6rem;" title="Pressure stall: the share of the last 10 s that tasks waited for the resource">
Pressure (10 s): cpu {{ pct(psi.cpu) }} · mem {{ pct(psi.mem) }} · io {{ pct(psi.io) }}
</p>
{% endif %}
</section>
</div>
{% endif %}
plugins/host_agent/templates/host_detail.html
+13 -13
View File
@@ -54,30 +54,30 @@
interaction: { mode: "index", intersect: false },
elements: { point: { radius: 0 } },
scales: {
x: { type: "linear", ticks: { callback: (v) => fmtTime(v), maxTicksLimit: 8, font: { size: 10 } } },
y: { beginAtZero: true, max: ymax, ticks: { font: { size: 10 } } },
x: { type: "linear", ticks: { callback: (v) => fmtTime(v), maxTicksLimit: 8, color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
y: { beginAtZero: true, max: ymax, ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
},
plugins: {
legend: { labels: { boxWidth: 10, font: { size: 11 } } },
legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } },
tooltip: { callbacks: { title: (items) => items.length ? fmtTime(items[0].parsed.x) : "" } },
},
});
const defs = {
"chart-util": { ymax: 100, ds: [
{ key: "cpu_pct", label: "CPU %", color: stewardColor("--gold") },
{ key: "mem_used_pct", label: "Mem %", color: stewardColor("--green") },
{ key: "disk_used_pct_worst", label: "Disk % (worst)", color: stewardColor("--yellow") },
{ key: "cpu_pct", label: "CPU %", color: "#c8a840" },
{ key: "mem_used_pct", label: "Mem %", color: "#4aa86a" },
{ key: "disk_used_pct_worst", label: "Disk % (worst)", color: "#c87840" },
] },
"chart-net": { ymax: undefined, ds: [
{ key: "net_rx_bps", label: "Net RX", color: stewardColor("--green") },
{ key: "net_tx_bps", label: "Net TX", color: stewardColor("--red") },
{ key: "disk_read_bps", label: "Disk read", color: stewardColor("--blue") },
{ key: "disk_write_bps", label: "Disk write", color: stewardColor("--gold") },
{ key: "net_rx_bps", label: "Net RX", color: "#4aa86a" },
{ key: "net_tx_bps", label: "Net TX", color: "#c84048" },
{ key: "disk_read_bps", label: "Disk read", color: "#6aa0d0" },
{ key: "disk_write_bps", label: "Disk write", color: "#c8a840" },
] },
"chart-pressure": { ymax: undefined, ds: [
{ key: "load_1m", label: "Load 1m", color: stewardColor("--gold") },
{ key: "psi_mem_some_avg10", label: "Mem PSI %", color: stewardColor("--red") },
{ key: "temp_c_max", label: "Temp °C", color: stewardColor("--yellow") },
{ key: "load_1m", label: "Load 1m", color: "#c8a840" },
{ key: "psi_mem_some_avg10", label: "Mem PSI %", color: "#c84048" },
{ key: "temp_c_max", label: "Temp °C", color: "#c87840" },
] },
};
const charts = {};
plugins/host_agent/templates/install.sh.j2
-4
View File
@@ -45,10 +45,6 @@ cat > "$CONF_FILE" <<EOF
url = $STEWARD_URL
token = $AGENT_TOKEN
interval_seconds = 30
# Container logs are collected by default. To opt this host out entirely:
# docker_logs_enabled = false
# To skip specific noisy containers (comma-separated names):
# docker_log_exclude = watchtower, some-chatty-service
EOF
chown "root:$AGENT_USER" "$CONF_FILE"
chmod 0640 "$CONF_FILE"
plugins/host_agent/templates/panel.html
+51 -65
View File
@@ -4,9 +4,9 @@
<div style="display:flex;align-items:baseline;justify-content:space-between;margin-bottom:0.75rem;gap:1rem;flex-wrap:wrap;">
<h3 class="section-title" style="margin-bottom:0;">Agent</h3>
{% if reporting %}
<span style="font-size:0.78rem;" class="tone-{{ 'warn' if stale else 'ok' }}">
<span style="font-size:0.78rem;color:{{ 'var(--yellow)' if stale else 'var(--green)' }};">
{{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %}
· last seen {{ reg.last_seen_at|ago }}
· last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }}
</span>
{% elif reg %}
<span style="font-size:0.78rem;color:var(--text-muted);">pending — no check-in yet</span>
@@ -15,57 +15,52 @@
{% if reporting %}
{# Reporting: live vitals are shown by the vitals strip above; this panel is
the way to the full metrics, with the lifecycle actions folded under
"Manage agent" (#5278). #}
<a href="/plugins/host_agent/{{ host.id }}/" class="btn btn-sm btn-ghost">Full metrics →</a>
lifecycle/management only. #}
<div style="display:flex;gap:0.5rem;flex-wrap:wrap;align-items:center;">
<a href="/plugins/host_agent/{{ host.id }}/" class="btn btn-sm btn-ghost">Full metrics →</a>
{% if session.user_role == 'admin' %}
{% if ansible_available and target %}
<form method="post" action="/plugins/host_agent/update" style="margin:0;">
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<button type="submit" class="btn btn-sm" title="Refresh agent.py + restart (token preserved)">Update agent</button>
</form>
{% endif %}
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/rotate-token" style="margin:0;"
onsubmit="return confirm('Rotate token? The agent stops reporting until reinstalled/updated with the new token.');">
<button type="submit" class="btn btn-sm btn-ghost">Rotate token</button>
</form>
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0;"
onsubmit="return confirm('Remove this agent registration? Metrics stop until re-registered.');">
<button type="submit" class="btn btn-sm btn-danger">Remove</button>
</form>
{% endif %}
</div>
{% if ansible_available and not target %}
<p style="color:var(--text-dim);font-size:0.8rem;margin:0.6rem 0 0;">
Link an Ansible target (above) to enable one-click agent updates.
</p>
{% endif %}
{% if session.user_role == 'admin' %}
{% if session.user_role == 'admin' and ansible_available and target and managed_key_set %}
<details style="margin-top:0.7rem;">
<summary style="cursor:pointer;font-size:0.82rem;color:var(--text-muted);">Manage agent</summary>
<div style="display:flex;gap:0.5rem;flex-wrap:wrap;align-items:center;margin-top:0.6rem;">
{% if ansible_available and target %}
<form method="post" action="/plugins/host_agent/update" style="margin:0;">{{ csrf_field() }}
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<button type="submit" class="btn btn-sm" title="Refresh agent.py and restart it; the token is kept">Update agent</button>
</form>
{% endif %}
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/rotate-token" style="margin:0;"
data-confirm="Rotate token? The agent stops reporting until reinstalled/updated with the new token.">{{ csrf_field() }}
<button type="submit" class="btn btn-sm btn-ghost">Rotate token</button>
</form>
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0;"
data-confirm="Unregister this host's agent? Its token stops working and metrics stop until it is registered again. The agent itself stays installed on the host.">{{ csrf_field() }}
<button type="submit" class="btn btn-sm btn-danger">Unregister agent</button>
</form>
</div>
{% if ansible_available and not target %}
<p style="color:var(--text-dim);font-size:0.8rem;margin:0.6rem 0 0;">
Link an Ansible target (below) to update the agent from here.
<summary style="cursor:pointer;font-size:0.8rem;color:var(--text-muted);">Re-provision (reinstall the steward account + managed key, then the agent)</summary>
<p style="font-size:0.78rem;color:var(--text-dim);margin:0.5rem 0;">
Use after regenerating the managed key, or if SSH auth as <code>steward</code> breaks.
Connects over a one-time bootstrap user + password (not stored).
</p>
{% endif %}
{% if ansible_available and target and managed_key_set %}
<details style="margin-top:0.7rem;">
<summary style="cursor:pointer;font-size:0.8rem;color:var(--text-muted);">Re-provision (reinstall the steward account + managed key, then the agent)</summary>
<p style="font-size:0.78rem;color:var(--text-dim);margin:0.5rem 0;">
Use after regenerating the managed key, or if SSH auth as <code>steward</code> breaks.
Connects over a one-time bootstrap user + password (not stored).
</p>
<form method="post" action="/plugins/host_agent/provision"
style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<div class="form-group" style="margin:0;">
<label>Bootstrap user</label>
<input type="text" name="bootstrap_user" required placeholder="root" style="width:8rem;" autocomplete="off">
</div>
<div class="form-group" style="margin:0;">
<label>Bootstrap password</label>
<input type="password" name="bootstrap_password" required style="width:10rem;" autocomplete="new-password">
</div>
<button type="submit" class="btn btn-sm">Re-provision</button>
</form>
</details>
{% endif %}
<form method="post" action="/plugins/host_agent/provision"
style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;">
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<div class="form-group" style="margin:0;">
<label>Bootstrap user</label>
<input type="text" name="bootstrap_user" required placeholder="root" style="width:8rem;" autocomplete="off">
</div>
<div class="form-group" style="margin:0;">
<label>Bootstrap password</label>
<input type="password" name="bootstrap_password" required style="width:10rem;" autocomplete="new-password">
</div>
<button type="submit" class="btn btn-sm">Re-provision</button>
</form>
</details>
{% endif %}
@@ -91,7 +86,8 @@
{% elif not ansible_available %}
<p style="color:var(--text-dim);font-size:0.85rem;">
Ansible isn't available, so the agent can't be deployed from here. Install it manually instead (below).
Ansible isn't available, so the agent can't be deployed from here. Install it manually with the
<a href="/plugins/host_agent/fleet/">curl install command</a>.
</p>
{% elif not managed_key_set %}
@@ -100,7 +96,7 @@
border-radius:6px;padding:0.7rem 0.9rem;display:flex;align-items:center;gap:0.8rem;flex-wrap:wrap;">
<span style="color:var(--yellow);">⚠</span>
<span style="font-size:0.84rem;flex:1;min-width:13rem;">No managed SSH key yet — Steward needs one to log into hosts.</span>
<form method="post" action="/settings/ansible/generate-key" style="margin:0;">{{ csrf_field() }}
<form method="post" action="/settings/ansible/generate-key" style="margin:0;">
<input type="hidden" name="next" value="/hosts/{{ host.id }}">
<button type="submit" class="btn btn-sm">Generate managed key</button>
</form>
@@ -119,7 +115,7 @@
</p>
{# Provision: brand-new host (creates steward account + key over a one-time password) #}
<form method="post" action="/plugins/host_agent/provision"
style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;margin-bottom:0.75rem;">{{ csrf_field() }}
style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;margin-bottom:0.75rem;">
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<div style="font-size:0.78rem;color:var(--text-muted);width:100%;">Provision (first contact — fresh host):</div>
<div class="form-group" style="margin:0;">
@@ -133,26 +129,16 @@
<button type="submit" class="btn btn-sm">Provision</button>
</form>
{# Install: host already has the steward account (managed key works) #}
<form method="post" action="/plugins/host_agent/deploy" style="display:flex;gap:0.6rem;align-items:center;">{{ csrf_field() }}
<form method="post" action="/plugins/host_agent/deploy" style="display:flex;gap:0.6rem;align-items:center;">
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<span style="font-size:0.78rem;color:var(--text-muted);">Already provisioned?</span>
<button type="submit" class="btn btn-sm btn-ghost">Install agent (managed key)</button>
</form>
{% endif %}
{% if not reg and session.user_role == 'admin' %}
{# A manual install: no Ansible, no SSH. Issues this host's token and shows
the one-line install command on the fleet page, once (#5282). #}
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/register"
style="display:flex;gap:0.6rem;align-items:center;flex-wrap:wrap;margin:0.75rem 0 0;">{{ csrf_field() }}
<span style="font-size:0.78rem;color:var(--text-muted);">Or by hand on the host:</span>
<button type="submit" class="btn btn-sm btn-ghost">Install manually</button>
</form>
{% endif %}
{% if reg and session.user_role == 'admin' %}
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0.75rem 0 0;"
data-confirm="Clear this pending registration? Its token is discarded.">{{ csrf_field() }}
onsubmit="return confirm('Clear this pending registration? Its token is discarded.');">
<button type="submit" class="btn btn-sm btn-ghost">Clear pending registration</button>
</form>
{% endif %}
plugins/host_agent/templates/settings_list.html
+57 -47
View File
@@ -1,5 +1,5 @@
{% extends "base.html" %}
{% from "_macros.html" import crumbs, status_badge %}
{% from "_macros.html" import crumbs %}
{% block title %}Agent fleet — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), ("Agent fleet", "")]) }}{% endblock %}
{% block content %}
@@ -8,12 +8,11 @@
</div>
<p style="color:var(--text-muted);font-size:0.84rem;margin:-0.4rem 0 1.25rem;">
Bulk agent operations across your inventory. For a single host, use its
<a href="/hosts/">host page</a>: provisioning, a manual install command and
metrics live there. New hosts are added with <a href="/hosts/?add=1">Add host</a>.
<a href="/hosts/">host page</a> — provisioning and metrics live there.
</p>
{% if new_token %}
<div class="card" style="background:var(--fs-accent-faint);">
<div class="card" style="background:var(--accent-bg);">
<h3>New token — copy this install command now</h3>
<p style="font-size:0.82rem;color:var(--text-muted);">
This is the only time the raw token is shown. Rotate if you lose it.
@@ -22,43 +21,19 @@
</div>
{% endif %}
<div class="card-flush" style="margin-bottom:1.5rem;">
<div class="table-wrap">
<table class="table">
<thead>
<tr>
<th>Status</th><th>Host</th><th>Last seen</th><th>Agent version</th><th>Reported IP</th><th>Distro</th>
<th class="td-actions">Actions</th>
</tr>
</thead>
<tbody>
{% for item in registrations %}
<tr>
<td>{{ status_badge(item.state, {'up': 'Reporting', 'stale': 'Silent', 'pending': 'Never reported'}[item.state]) }}</td>
<td>{% if item.host %}<a href="/hosts/{{ item.host.id }}">{{ item.host.name }}</a>{% else %}<span class="mono">{{ item.reg.host_id }}</span>{% endif %}</td>
<td>{% if item.reg.last_seen_at %}<span title="{{ item.reg.last_seen_at.strftime('%Y-%m-%d %H:%M:%S UTC') }}">{{ item.reg.last_seen_at|ago }}</span>{% else %}never{% endif %}</td>
<td>{{ item.reg.agent_version or "—" }}</td>
<td class="mono">{{ item.reg.host_ip or "—" }}</td>
<td>{{ item.reg.distro or "—" }}</td>
<td class="td-actions">
<form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/rotate-token"
style="display:inline;"
data-confirm="Rotate the token for {{ item.name }}? The existing agent will stop working until reinstalled with the new token.">{{ csrf_field() }}
<button type="submit" class="btn btn-ghost btn-sm">Rotate token</button>
</form>
<form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/delete"
style="display:inline;"
data-confirm="Unregister the agent for {{ item.name }}? Its token stops working and metrics stop until it is registered again. The agent itself stays installed on the host.">{{ csrf_field() }}
<button type="submit" class="btn btn-danger btn-sm">Unregister</button>
</form>
</td>
</tr>
{% else %}
<tr><td colspan="7" class="empty">No agents registered yet. <a href="/hosts/?add=1">Add a host</a>, then install its agent from the host's page.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
<div class="card">
<form method="post" action="/plugins/host_agent/fleet/add-host"
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
<div class="form-group" style="margin-bottom:0;">
<label>Host name</label>
<input type="text" name="name" required>
</div>
<div class="form-group" style="margin-bottom:0;">
<label>Address (optional)</label>
<input type="text" name="address">
</div>
<button type="submit" class="btn">Add host</button>
</form>
</div>
{% if ansible_available %}
@@ -74,7 +49,7 @@
<h3 style="margin-bottom:0.4rem;">Agent lifecycle via Ansible</h3>
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.6rem;">
These actions run bundled <strong>Ansible playbooks</strong> to deploy and maintain the
Steward monitoring agent on your <a href="/ansible/inventory/">inventory hosts</a>.
Steward monitoring agent on your <a href="/ansible/inventory/targets">inventory hosts</a>.
Steward generates each host's API token automatically and connects over SSH as the managed
<code>steward</code> account — you'll be dropped into the live Ansible run to watch it.
</p>
@@ -88,14 +63,14 @@
<span style="font-size:0.84rem;flex:1;min-width:14rem;">
No managed SSH key yet — Steward needs one to log into hosts as <code>steward</code>.
</span>
<form method="post" action="/settings/ansible/generate-key" style="margin:0;">{{ csrf_field() }}
<form method="post" action="/settings/ansible/generate-key" style="margin:0;">
<input type="hidden" name="next" value="/plugins/host_agent/fleet/">
<button type="submit" class="btn btn-sm">Generate managed key</button>
</form>
</div>
{% elif not (deploy_targets or deploy_groups) %}
<p style="font-size:0.85rem;color:var(--text-dim);margin:0;">
No Ansible inventory targets yet. Add some under <a href="/ansible/inventory/">Ansible → Inventory</a>.
No Ansible inventory targets yet. Add some under <a href="/ansible/inventory/targets">Ansible → Inventory</a>.
</p>
{% endif %}
</div>
@@ -110,7 +85,7 @@
agent. Run this once per host.
</p>
<form method="post" action="/plugins/host_agent/provision"
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
<div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
<div class="form-group" style="margin-bottom:0;">
<label>Bootstrap user</label>
@@ -136,7 +111,7 @@
mints a fresh token, and installs the agent. No <code>curl | sh</code> needed.
</p>
<form method="post" action="/plugins/host_agent/deploy"
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
<div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
<div class="form-group" style="margin-bottom:0;">
<label>Report interval (s)</label>
@@ -154,7 +129,7 @@
so identity is preserved.
</p>
<form method="post" action="/plugins/host_agent/update"
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
<div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
<button type="submit" class="btn">Update</button>
</form>
@@ -162,4 +137,39 @@
{% endif %}
{% endif %}
<div class="card-flush">
<table class="table">
<thead>
<tr>
<th>Host</th><th>Reported IP</th><th>Agent version</th><th>Distro</th>
<th>Last seen</th><th class="td-actions">Actions</th>
</tr>
</thead>
<tbody>
{% for item in registrations %}
<tr>
<td>{{ item.host.name if item.host else item.reg.host_id }}</td>
<td>{{ item.reg.host_ip or "—" }}</td>
<td>{{ item.reg.agent_version or "—" }}</td>
<td>{{ item.reg.distro or "—" }}</td>
<td>{{ item.reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") if item.reg.last_seen_at else "never" }}</td>
<td class="td-actions">
<form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/rotate-token"
style="display:inline;"
onsubmit="return confirm('Rotate the token for this host? The existing agent will stop working until reinstalled with the new token.');">
<button type="submit" class="btn btn-ghost btn-sm">Rotate token</button>
</form>
<form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/delete"
style="display:inline;"
onsubmit="return confirm('Delete this host registration? The agent will be unable to report metrics until re-registered.');">
<button type="submit" class="btn btn-danger btn-sm">Delete</button>
</form>
</td>
</tr>
{% else %}
<tr><td colspan="6" class="empty">No hosts registered. Add one above.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
{% endblock %}
plugins/host_agent/templates/widget_history.html
+15 -17
View File
@@ -16,14 +16,16 @@
toggle that re-requests this widget without entering edit mode. #}
<div style="flex:0 0 auto;display:flex;align-items:center;gap:0.5rem;flex-wrap:wrap;font-size:0.75rem;color:var(--text-muted);margin-bottom:0.3rem;">
{% if session.user_id %}
<a href="/hosts/{{ host.id }}" style="font-weight:500;">{{ host.name }}</a>
<a href="/hosts/{{ host.id }}" style="font-weight:600;">{{ host.name }}</a>
{% else %}
<strong style="color:var(--text);">{{ host.name }}</strong>
{% endif %}
<span class="hist-range" style="margin-left:auto;display:inline-flex;gap:0.25rem;">
<span class="hist-range" style="margin-left:auto;display:inline-flex;border:1px solid var(--border-mid);border-radius:5px;overflow:hidden;">
{% for opt in [1, 6, 24] %}
<button type="button" onclick="histRange({{ wid|int }}, {{ opt|int }})"
class="range-btn{% if opt == hours %} active{% endif %}">{{ opt }}h</button>
<button type="button" onclick="histRange({{ wid }}, '{{ host.id }}', {{ opt }})"
style="border:0;cursor:pointer;padding:0.1rem 0.45rem;font-size:0.72rem;
background:{% if opt == hours %}var(--accent){% else %}transparent{% endif %};
color:{% if opt == hours %}#fff{% else %}var(--text-muted){% endif %};">{{ opt }}h</button>
{% endfor %}
</span>
</div>
@@ -34,15 +36,11 @@
<script>
// Live time-range switch. Rewrites the parent cell's hx-get so the choice sticks
// across polls (htmx re-reads the attribute each poll), then fetches immediately.
// Only `hours` changes: the rest of the query (host, widget id, a share link's
// token) is kept, so the switch also works on a shared dashboard.
window.histRange = function (wid, hours) {
window.histRange = function (wid, hostId, hours) {
var cell = document.getElementById("widget-" + wid);
if (!cell) return;
var parts = (cell.getAttribute("hx-get") || "").split("?");
var query = new URLSearchParams(parts[1] || "");
query.set("hours", hours);
var url = parts[0] + "?" + query.toString();
var base = (cell.getAttribute("hx-get") || "").split("?")[0];
var url = base + "?host_id=" + encodeURIComponent(hostId) + "&hours=" + hours + "&wid=" + wid;
cell.setAttribute("hx-get", url);
if (window.htmx) window.htmx.ajax("GET", url, { target: cell, swap: "innerHTML" });
};
@@ -55,9 +53,9 @@ window.histRange = function (wid, hours) {
return ("0" + d.getHours()).slice(-2) + ":" + ("0" + d.getMinutes()).slice(-2);
};
var ds = [
{ key: "cpu_pct", label: "CPU %", color: stewardColor("--gold") },
{ key: "mem_used_pct", label: "Mem %", color: stewardColor("--green") },
{ key: "disk_root", label: "Disk / %", color: stewardColor("--yellow") },
{ key: "cpu_pct", label: "CPU %", color: "#c8a840" },
{ key: "mem_used_pct", label: "Mem %", color: "#4aa86a" },
{ key: "disk_root", label: "Disk / %", color: "#c87840" },
];
new Chart(el.getContext("2d"), {
type: "line",
@@ -73,14 +71,14 @@ window.histRange = function (wid, hours) {
interaction: { mode: "index", intersect: false },
elements: { point: { radius: 0 } },
scales: {
x: { type: "linear", ticks: { callback: function (v) { return fmtTime(v); }, maxTicksLimit: 6, font: { size: 10 } } },
x: { type: "linear", ticks: { callback: function (v) { return fmtTime(v); }, maxTicksLimit: 6, color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
// Fixed 0–{{ y_max }} ceiling (server snaps the data peak up to the next
// 10% band). Steady across refreshes — auto-scaling made the zoom jump on
// every poll — while a banded ceiling still keeps the lines filling the panel.
y: { min: 0, max: {{ y_max }}, ticks: { font: { size: 10 } } },
y: { min: 0, max: {{ y_max }}, ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
},
plugins: {
legend: { labels: { boxWidth: 10, font: { size: 11 } } },
legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } },
tooltip: { callbacks: { title: function (items) { return items.length ? fmtTime(items[0].parsed.x) : ""; } } },
},
},
plugins/host_agent/templates/widget_table.html
+77
View File
@@ -0,0 +1,77 @@
{# host_agent widget — fleet glance. One horizontal row per host, zebra-striped
so adjacent hosts read as distinct groupings: health dot + host name on the
left, metric cells (cpu/mem/disk/load) each with a trend sparkline laid out
horizontally on the right. The name wraps (never hard-truncated) but the row
stays horizontal. #}
{% if rows %}
{% macro metric_cell(label, value, fmt, svg, style="", title="") %}
<div style="min-width:62px;flex:0 0 auto;" title="{{ title }}">
<div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
<div style="font-weight:600;font-size:0.8rem;font-variant-numeric:tabular-nums;{{ style }}">
{% if value is not none %}{{ fmt|format(value) }}{% else %}—{% endif %}
</div>
<div style="line-height:0;height:16px;">{{ svg | safe }}</div>
</div>
{% endmacro %}
{# Human-readable throughput, matching the host-detail page. #}
{% macro fmt_bps(v) %}
{%- if v is none -%}—
{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s
{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s
{%- else -%}{{ "%.0f"|format(v) }} B/s
{%- endif -%}
{% endmacro %}
{# Two-line down/up (or read/write) throughput cell. #}
{% macro io_cell(label, down_sym, down_val, up_sym, up_val, title="") %}
<div style="min-width:80px;flex:0 0 auto;" title="{{ title }}">
<div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
<div style="font-size:0.72rem;font-variant-numeric:tabular-nums;line-height:1.3;">
<div>{{ down_sym }} {{ fmt_bps(down_val) }}</div>
<div>{{ up_sym }} {{ fmt_bps(up_val) }}</div>
</div>
</div>
{% endmacro %}
<div class="host-fleet">
{% for r in rows %}
{% set stale = r.stale %}
<div style="display:flex;align-items:center;gap:0.6rem 1rem;flex-wrap:wrap;padding:0.5rem 0.6rem;border-radius:4px;{% if loop.index0 % 2 %}background:var(--bg-elevated);{% endif %}">
{# Left — health dot + full host name (wraps, never truncated) + last seen #}
<div style="display:flex;align-items:center;gap:0.5rem;flex:1 1 150px;min-width:120px;">
<span class="dot {% if stale %}dot-dim{% elif (r.cpu_pct or 0) >= 90 or (r.mem_used_pct or 0) >= 90 or (r.disk_worst or 0) >= 90 %}dot-warn{% else %}dot-up{% endif %}"
style="flex-shrink:0;"
title="{% if stale %}stale / no recent data{% else %}warns at CPU/memory ≥90% or any disk mount ≥90% (the 'disk /' figure shows root only){% endif %}"></span>
<span style="min-width:0;">
{% if session.user_id %}
<a href="/hosts/{{ r.host.id }}" style="font-weight:600;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</a>
{% else %}
<span style="font-weight:600;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</span>
{% endif %}
<span style="display:block;font-size:0.7rem;color:var(--text-dim);">
{{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }}
</span>
</span>
</div>
{# Right — metric cells with value + trend, laid out horizontally #}
<div style="display:flex;flex-wrap:wrap;gap:0.4rem 1rem;justify-content:flex-end;">
{{ metric_cell("cpu", r.cpu_pct, "%.0f%%", r.sparks.cpu, threshold_style(r.cpu_pct, 'cpu'), "Average CPU utilization") }}
{{ metric_cell("mem", r.mem_used_pct, "%.0f%%", r.sparks.mem, threshold_style(r.mem_used_pct, 'mem'), "Memory in use") }}
{{ metric_cell("disk /", r.disk_root, "%.0f%%", r.sparks.disk, threshold_style(r.disk_root, 'disk'), "Root filesystem (/) usage") }}
{{ metric_cell("load", r.load_1m, "%.2f", r.sparks.load, "", "Load average (1m)") }}
{# Extra agent metrics — shown only when the host reports them (VMs/containers
often have no temp sensor, etc.) so absent data doesn't clutter the row. #}
{% if r.net_rx_bps is not none or r.net_tx_bps is not none %}
{{ io_cell("net", "↓", r.net_rx_bps, "↑", r.net_tx_bps, "Network throughput (down / up)") }}
{% endif %}
{% if r.disk_read_bps is not none or r.disk_write_bps is not none %}
{{ io_cell("disk i/o", "rd", r.disk_read_bps, "wr", r.disk_write_bps, "Disk I/O (read / write)") }}
{% endif %}
{% if r.temp_max is not none %}
{{ metric_cell("temp", r.temp_max, "%.0f°C", "", threshold_style(r.temp_max, 'temp'), "Hottest sensor") }}
{% endif %}
</div>
</div>
{% endfor %}
</div>
{% else %}
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No hosts with agent data yet.</p>
{% endif %}
plugins/index.yaml
+74
View File
@@ -0,0 +1,74 @@
# steward-plugins / index.yaml
#
# Plugin catalog for Steward.
# Fetched by the app at: Settings → Plugins → Plugin Catalog
#
# After updating an entry, commit and push — changes are live within 5 minutes
# (the app caches this file in-process for 300 seconds).
#
# See docs/plugins/index.yaml.example in the main app repo for the full schema.
version: 1
updated: "2026-03-22"
plugins:
- name: docker
version: "2.0.0"
description: "Per-host Docker container status + resource usage, collected by the Steward host agent"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v2.0.0/docker.zip"
checksum_sha256: ""
tags:
- containers
- docker
- infrastructure
- name: traefik
version: "1.0.0"
description: "Traefik reverse proxy metrics and access log integration"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip"
checksum_sha256: ""
tags:
- proxy
- metrics
- access-log
- name: unifi
version: "1.0.0"
description: "UniFi Network controller integration — WAN health, devices, clients, DPI"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/unifi-v1.0.0/unifi.zip"
checksum_sha256: ""
tags:
- network
- unifi
- ubiquiti
- name: ups
version: "1.0.0"
description: "UPS monitoring via NUT (Network UPS Tools) with Ansible shutdown automation"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/ups"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/ups-v1.0.0/ups.zip"
checksum_sha256: ""
tags:
- ups
- power
- nut
plugins/snmp/__init__.py
-13
View File
@@ -11,7 +11,6 @@ _app: "Quart | None" = None
def setup(app: "Quart") -> None:
global _app
_app = app
from .models import SnmpDevice # noqa: F401 — registers the table
def get_scheduled_tasks() -> list:
@@ -27,15 +26,3 @@ def get_blueprint():
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (SNMP device readings).
return [{"label": "SNMP", "href": "/plugins/snmp/"}]
def get_alert_metrics() -> dict[str, list[str]]:
"""SNMP metric names are the OID labels each device is configured with, so
the list is empty here and the rule form offers the labels recorded so far."""
return {"snmp": []}
def get_audit_entities() -> dict:
"""Audit rows about a device name it and link to its page."""
from .models import SnmpDevice
return {"snmp_device": (SnmpDevice, "/plugins/snmp/devices/{}")}
plugins/snmp/migrations/__init__.py
View File
plugins/snmp/migrations/versions/__init__.py
View File
plugins/snmp/migrations/versions/snmp_001_devices.py
-49
View File
@@ -1,49 +0,0 @@
"""SNMP devices are rows edited on the SNMP page
Devices lived in plugin.yaml's config, which the UI could not edit, and a poll
that failed left no trace. A row per device holds what to ask and how, and the
newest poll's outcome (milestone 491).
Revision ID: snmp_001_devices
Revises: (none — a branch off core via depends_on)
Create Date: 2026-10-08
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "snmp_001_devices"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = "snmp"
depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
def upgrade() -> None:
op.create_table(
"snmp_devices",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("name", sa.String(100), nullable=False, unique=True),
sa.Column("address", sa.String(255), nullable=False),
sa.Column("port", sa.Integer, nullable=False, server_default="161"),
sa.Column("version", sa.String(3), nullable=False, server_default="2c"),
sa.Column("community", sa.Text, nullable=False, server_default=""),
sa.Column("v3_user", sa.String(100), nullable=False, server_default=""),
sa.Column("v3_auth", sa.String(16), nullable=False, server_default="none"),
sa.Column("v3_auth_key", sa.Text, nullable=False, server_default=""),
sa.Column("v3_priv", sa.String(16), nullable=False, server_default="none"),
sa.Column("v3_priv_key", sa.Text, nullable=False, server_default=""),
sa.Column("presets", sa.JSON, nullable=False, server_default="[]"),
sa.Column("interfaces", sa.String(255), nullable=False, server_default=""),
sa.Column("custom", sa.Text, nullable=False, server_default=""),
sa.Column("host_id", sa.String(36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("last_polled_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_ok_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text, nullable=True),
sa.Column("missing", sa.JSON, nullable=False, server_default="[]"),
)
def downgrade() -> None:
op.drop_table("snmp_devices")
plugins/snmp/models.py
-42
View File
@@ -1,42 +0,0 @@
# plugins/snmp/models.py
from __future__ import annotations
from datetime import datetime, timezone
from sqlalchemy import JSON, DateTime, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from steward.core.ids import time_ordered_id
from steward.models.base import Base
class SnmpDevice(Base):
"""A device Steward polls over SNMP, edited on the SNMP page.
Secrets (the community, the v3 keys) are stored encrypted. The last_*
columns are the newest poll's outcome, written by the poller, so a page
can say why a device isn't answering without asking the device itself.
"""
__tablename__ = "snmp_devices"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
address: Mapped[str] = mapped_column(String(255), nullable=False)
port: Mapped[int] = mapped_column(Integer, nullable=False, default=161)
version: Mapped[str] = mapped_column(String(3), nullable=False, default="2c")
community: Mapped[str] = mapped_column(Text, nullable=False, default="")
v3_user: Mapped[str] = mapped_column(String(100), nullable=False, default="")
v3_auth: Mapped[str] = mapped_column(String(16), nullable=False, default="none")
v3_auth_key: Mapped[str] = mapped_column(Text, nullable=False, default="")
v3_priv: Mapped[str] = mapped_column(String(16), nullable=False, default="none")
v3_priv_key: Mapped[str] = mapped_column(Text, nullable=False, default="")
presets: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
interfaces: Mapped[str] = mapped_column(String(255), nullable=False, default="")
custom: Mapped[str] = mapped_column(Text, nullable=False, default="")
host_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
last_polled_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_ok_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
missing: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
plugins/snmp/plugin.yaml
+45 -3
View File
@@ -1,7 +1,7 @@
# plugins/snmp/plugin.yaml
name: snmp
version: "2.0.0"
description: "SNMP polling for network devices — switches, routers, printers, UPSes, servers."
version: "1.0.0"
description: "SNMP polling for network devices — switches, routers, printers, UPSes, etc."
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
@@ -12,6 +12,48 @@ tags:
- network
- monitoring
# Devices are added, tested and edited on the SNMP page (/plugins/snmp/).
config:
poll_interval_seconds: 60
# Each device is polled independently. OIDs must resolve to numeric values.
#
# `host` is the SNMP poll target (IP/hostname we query). A device also shows
# up on a Steward Host's detail page when it maps to that host. By default
# that mapping is implicit: `host` is matched against the Host's address or
# name. To bind explicitly instead — e.g. when you poll by IP but the Host is
# recorded by DNS name, or for an SNMP-only switch/PDU/UPS — add ONE of:
# host_id: "<steward-host-uuid>" # exact Host.id match
# steward_host: "switch01" # Host name OR address (case-insensitive)
# An explicit binding is exclusive: the device then maps ONLY to that host,
# never implicitly to another by a coincidental address string.
devices:
- name: "core-switch"
host: "192.168.1.1"
port: 161
community: "public"
version: "2c" # "1", "2c", or "3"
oids:
- oid: "1.3.6.1.2.1.1.3.0"
label: "uptime_centisecs"
- oid: "1.3.6.1.2.1.2.2.1.10.1"
label: "if1_in_octets"
- oid: "1.3.6.1.2.1.2.2.1.16.1"
label: "if1_out_octets"
# Example: network UPS with SNMP management card (RFC 1628 UPS-MIB).
# upsBatteryStatus values: 1=unknown, 2=batteryNormal, 3=batteryLow, 4=batteryDepleted.
# Uncomment and point at your UPS to enable.
# - name: "ups"
# host: "192.168.1.10"
# port: 161
# community: "public"
# version: "2c"
# oids:
# - oid: "1.3.6.1.2.1.33.1.2.1.0"
# label: "battery_status"
# - oid: "1.3.6.1.2.1.33.1.2.2.0"
# label: "seconds_on_battery"
# - oid: "1.3.6.1.2.1.33.1.2.3.0"
# label: "runtime_minutes_remaining"
# - oid: "1.3.6.1.2.1.33.1.2.4.0"
# label: "battery_charge_pct"
# - oid: "1.3.6.1.2.1.33.1.3.3.1.3.1"
# label: "input_voltage_dV"
plugins/snmp/poller.py
+102 -197
View File
@@ -1,53 +1,19 @@
# plugins/snmp/poller.py
"""Ask an SNMP device for its readings, and say in words why it didn't answer.
"""
Asynchronous SNMP GET helper.
pysnmp-lextudio 6.x (what the image installs) and canonical pysnmp 7.x put the
asyncio API in different modules with different names; _api() finds whichever
is installed, so a dependency bump can't silently break polling. Without
pysnmp at all the poll reports that instead of raising.
Requires pysnmp-lextudio (the maintained pysnmp fork), bundled into the Docker
image via the `snmp` extra (`pip install .[snmp]`):
pip install 'steward[snmp]'
If pysnmp is not installed, poll_device() returns an empty dict and logs a
warning — SNMP polling is then simply disabled, nothing else breaks.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass, field
from types import SimpleNamespace
logger = logging.getLogger(__name__)
TIMEOUT_SECONDS = 5
# OIDs per GET. A device's whole list usually fits in one request; this keeps
# a long custom list under the size an agent will answer.
OIDS_PER_REQUEST = 24
AUTH_PROTOCOLS = {"none": "usmNoAuthProtocol", "md5": "usmHMACMD5AuthProtocol",
"sha": "usmHMACSHAAuthProtocol", "sha224": "usmHMAC128SHA224AuthProtocol",
"sha256": "usmHMAC192SHA256AuthProtocol", "sha384": "usmHMAC256SHA384AuthProtocol",
"sha512": "usmHMAC384SHA512AuthProtocol"}
PRIV_PROTOCOLS = {"none": "usmNoPrivProtocol", "des": "usmDESPrivProtocol", "3des": "usm3DESEDEPrivProtocol",
"aes128": "usmAesCfb128Protocol", "aes192": "usmAesCfb192Protocol",
"aes256": "usmAesCfb256Protocol"}
# A varbind value of one of these types means the agent has no such object.
_ABSENT = {"NoSuchObject", "NoSuchInstance", "EndOfMibView"}
@dataclass
class Credentials:
version: str = "2c"
community: str = ""
user: str = ""
auth: str = "none"
auth_key: str = ""
priv: str = "none"
priv_key: str = ""
@dataclass
class PollResult:
readings: dict[str, float] = field(default_factory=dict)
error: str | None = None # the device didn't answer, and why
missing: list[str] = field(default_factory=list) # keys it answered without
def _pysnmp_available() -> bool:
try:
@@ -57,27 +23,9 @@ def _pysnmp_available() -> bool:
return False
def _api() -> SimpleNamespace:
"""The installed pysnmp's asyncio API under one set of names."""
try:
from pysnmp.hlapi.v3arch import asyncio as hl
from pysnmp.hlapi.v3arch.asyncio import auth
return SimpleNamespace(hl=hl, auth=auth, get=hl.get_cmd, walk=hl.walk_cmd, async_transport=True)
except ImportError:
from pysnmp.hlapi import asyncio as hl
from pysnmp.hlapi.asyncio import auth
return SimpleNamespace(hl=hl, auth=auth, get=hl.getCmd, walk=hl.walkCmd, async_transport=False)
def _auth_data(api, creds: Credentials):
if creds.version == "3":
return api.hl.UsmUserData(
creds.user,
authKey=creds.auth_key or None,
privKey=creds.priv_key or None,
authProtocol=getattr(api.auth, AUTH_PROTOCOLS.get(creds.auth, "usmNoAuthProtocol")),
privProtocol=getattr(api.auth, PRIV_PROTOCOLS.get(creds.priv, "usmNoPrivProtocol")))
return api.hl.CommunityData(creds.community, mpModel=0 if creds.version == "1" else 1)
def _mp_model(version: str) -> int:
"""Map an SNMP version string to pysnmp's mpModel int (0 = v1, 1 = v2c)."""
return 0 if version == "1" else 1
def _close_engine(engine) -> None:
@@ -118,149 +66,106 @@ def _close_engine(engine) -> None:
return
def explain(error_indication, address: str, port: int, creds: Credentials) -> str:
"""pysnmp's error indication, said as what to check."""
text = str(error_indication)
low = text.lower()
if "timeout" in low or "no snmp response" in low:
who = (f"the v3 user “{creds.user}”" if creds.version == "3"
else "the community")
return (f"No reply from {address}:{port} within {TIMEOUT_SECONDS} s. Check the address and port, "
f"that SNMP is on, and {who}: a device ignores a request it doesn't accept.")
if "unknown" in low and "user" in low:
return f"The device doesn't know the v3 user “{creds.user}”."
if "digest" in low or "authentication" in low:
return "The device refused the v3 authentication: check the auth protocol and key."
if "decrypt" in low or "privacy" in low:
return "The device couldn't decrypt the request: check the privacy protocol and key."
if "not in time window" in low:
return "The device rejected the request as out of its time window; try again."
return f"SNMP error from {address}:{port}: {text}"
async def poll_device(
host: str,
port: int,
community: str,
version: str,
oids: list[dict],
) -> dict[str, float]:
"""Perform an SNMP GET for each OID and return ``{label: float_value}``.
Non-numeric OIDs (strings, etc.) are skipped. Returns an empty dict on any
error (unreachable host, wrong community, …) so a flaky device never breaks
the poll loop.
async def _transport(api, address: str, port: int):
if api.async_transport:
return await api.hl.UdpTransportTarget.create((address, port), timeout=TIMEOUT_SECONDS, retries=1)
return api.hl.UdpTransportTarget((address, port), timeout=TIMEOUT_SECONDS, retries=1)
async def poll_device(address: str, port: int, creds: Credentials,
oids: list[tuple[str, str]]) -> PollResult:
"""GET every (oid, key) of a device, a few dozen to a request.
Returns the numeric answers by key, the keys the device answered without
(no such object, or not a number), and an error in words when it didn't
answer at all. Never raises: a broken device must not stop the others.
pysnmp's HLAPI is asyncio-only as of v6. The import path moved between major
versions, so we support both rather than let a dependency bump silently
re-break polling:
• pysnmp-lextudio 6.2.x → ``pysnmp.hlapi.asyncio`` (``getCmd`` + a directly
constructed ``UdpTransportTarget``).
• canonical pysnmp 7.x → ``pysnmp.hlapi.v3arch.asyncio`` (``get_cmd`` + the
async ``UdpTransportTarget.create``).
"""
result = PollResult()
if not oids:
return result
if not _pysnmp_available():
result.error = "pysnmp isn't installed in this image, so SNMP polling is off."
return result
logger.warning("pysnmp not installed — SNMP polling disabled. "
"Install with: pip install 'steward[snmp]'")
return {}
api = _api()
engine = api.hl.SnmpEngine()
try:
try:
transport = await _transport(api, address, port)
except Exception as exc:
result.error = f"Can't reach {address}:{port}: {exc}"
return result
auth = _auth_data(api, creds)
# canonical pysnmp 7.x
from pysnmp.hlapi.v3arch.asyncio import (
CommunityData,
ContextData,
ObjectIdentity,
ObjectType,
SnmpEngine,
UdpTransportTarget,
get_cmd as _get_cmd,
)
_transport_is_async = True
except ImportError:
# pysnmp-lextudio 6.2.x
from pysnmp.hlapi.asyncio import (
CommunityData,
ContextData,
ObjectIdentity,
ObjectType,
SnmpEngine,
UdpTransportTarget,
getCmd as _get_cmd,
)
_transport_is_async = False
pending = list(oids)
while pending:
chunk, pending = pending[:OIDS_PER_REQUEST], pending[OIDS_PER_REQUEST:]
ei, es, eidx, var_binds = await api.get(
engine, auth, transport, api.hl.ContextData(),
*[api.hl.ObjectType(api.hl.ObjectIdentity(oid)) for oid, _ in chunk],
lookupMib=False)
if ei:
result.error = explain(ei, address, port, creds)
return result
if es:
# v1 fails the whole request on the first unknown OID: note it,
# and ask again for the rest.
bad = int(eidx) - 1 if eidx else -1
if 0 <= bad < len(chunk):
result.missing.append(chunk[bad][1])
pending = chunk[:bad] + chunk[bad + 1:] + pending
continue
result.error = f"The device answered with an error: {es.prettyPrint()}"
return result
# A GET answers in the order it asked.
for (_, key), (_, val) in zip(chunk, var_binds):
if type(val).__name__ in _ABSENT:
result.missing.append(key)
continue
engine = SnmpEngine()
# Always release the engine's UDP socket — see _close_engine for why.
try:
# Same host/port for every OID on this device, so build the transport once.
try:
if _transport_is_async:
transport = await UdpTransportTarget.create((host, port), timeout=5, retries=1)
else:
transport = UdpTransportTarget((host, port), timeout=5, retries=1)
except Exception as exc:
logger.debug("SNMP transport setup failed for %s:%s: %s", host, port, exc)
return {}
results: dict[str, float] = {}
for oid_cfg in oids:
oid = oid_cfg["oid"]
label = oid_cfg.get("label") or oid
scale = float(oid_cfg.get("scale", 1.0))
try:
error_indication, error_status, error_index, var_binds = await _get_cmd(
engine,
CommunityData(community, mpModel=_mp_model(version)),
transport,
ContextData(),
ObjectType(ObjectIdentity(oid)),
)
except Exception as exc:
logger.debug("SNMP GET %s@%s OID %s failed: %s", host, port, oid, exc)
continue
if error_indication:
logger.debug("SNMP error %s@%s OID %s: %s", host, port, oid, error_indication)
continue
if error_status:
logger.debug("SNMP status %s@%s OID %s: %s at %s",
host, port, oid, error_status.prettyPrint(),
error_index and var_binds[int(error_index) - 1][0] or "?")
continue
for _, val in var_binds:
try:
result.readings[key] = float(val)
results[label] = float(val) * scale
except (TypeError, ValueError):
result.missing.append(key)
if not result.readings and result.missing:
result.error = "The device answered but has none of these readings."
return result
except Exception as exc:
logger.warning("SNMP poll of %s:%s failed: %s", address, port, exc)
result.error = f"SNMP poll failed: {exc}"
return result
finally:
_close_engine(engine)
# Non-numeric type (e.g. OctetString description) — skip.
logger.debug("SNMP non-numeric value for %s label=%s: %r", oid, label, val)
SYS_DESCR, SYS_UPTIME, SYS_NAME = "1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"
IF_DESCR, IF_NAME = "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.31.1.1.1.1"
INTERFACES_LISTED = 128
async def describe_device(address: str, port: int, creds: Credentials) -> dict:
"""What Test shows: {"error"} or {"name", "description", "uptime_cs",
"interfaces": [(number, name)]}. Asks the device directly, so only an
operator's explicit Test calls it, never a page load."""
if not _pysnmp_available():
return {"error": "pysnmp isn't installed in this image, so SNMP is off."}
api = _api()
engine = api.hl.SnmpEngine()
try:
try:
transport = await _transport(api, address, port)
except Exception as exc:
return {"error": f"Can't reach {address}:{port}: {exc}"}
auth = _auth_data(api, creds)
ei, es, _, var_binds = await api.get(
engine, auth, transport, api.hl.ContextData(),
*[api.hl.ObjectType(api.hl.ObjectIdentity(o)) for o in (SYS_NAME, SYS_DESCR, SYS_UPTIME)],
lookupMib=False)
if ei:
return {"error": explain(ei, address, port, creds)}
if es:
return {"error": f"The device answered with an error: {es.prettyPrint()}"}
values = [None if type(v).__name__ in _ABSENT else v for _, v in var_binds]
out = {"name": str(values[0]) if values[0] is not None else "",
"description": str(values[1]) if values[1] is not None else "",
"uptime_cs": float(values[2]) if values[2] is not None else None,
"interfaces": []}
# ifName where the device has it (IF-MIB), else ifDescr.
for column in (IF_NAME, IF_DESCR):
found: list[tuple[int, str]] = []
async for ei, es, _, rows in api.walk(engine, auth, transport, api.hl.ContextData(),
api.hl.ObjectType(api.hl.ObjectIdentity(column)),
lexicographicMode=False, lookupMib=False):
if ei or es:
break
for name, val in rows:
if type(val).__name__ in _ABSENT:
continue
found.append((int(str(name).rsplit(".", 1)[-1]), str(val)))
if len(found) >= INTERFACES_LISTED:
break
if found:
out["interfaces"] = found
break
return out
except Exception as exc:
logger.warning("SNMP test of %s:%s failed: %s", address, port, exc)
return {"error": f"SNMP test failed: {exc}"}
return results
finally:
_close_engine(engine)
plugins/snmp/readings.py
-217
View File
@@ -1,217 +0,0 @@
# plugins/snmp/readings.py
"""What an SNMP device is asked for, and how each answer reads.
A device's readings come from the presets ticked on its form plus any custom
lines. Each reading has a key (the metric name it is stored and alerted on),
a name a person reads, and a unit. A counter (an interface's byte count, any
"total so far") is stored as a rate: the poller turns two samples into a
per-second figure, so a chart and an alert see traffic, not a lifetime total.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from steward.core.formatting import human_bytes, human_duration
@dataclass(frozen=True)
class Reading:
oid: str
key: str
name: str
unit: str = "number"
scale: float = 1.0
offset: float = 0.0
@property
def counter(self) -> bool:
return self.unit in COUNTER_UNITS
# A counter's unit names what it counts; it is stored as the rate unit it maps
# to, multiplied by the factor (a byte counter is stored as bits per second).
COUNTER_UNITS = {"octets": ("bits_per_second", 8.0), "count": ("per_second", 1.0)}
STATUS_WORDS = {
"if_status": {1: "up", 2: "down", 3: "testing", 4: "unknown", 5: "dormant",
6: "not present", 7: "lower layer down"},
"ups_battery": {1: "unknown", 2: "normal", 3: "low", 4: "depleted"},
}
# The units a custom line may name, with how each one reads, for the form's hint.
UNITS = {
"number": "a plain number",
"percent": "0 to 100",
"celsius": "degrees Celsius",
"volts": "volts",
"decivolts": "tenths of a volt",
"seconds": "a span in seconds",
"minutes": "a span in minutes",
"centiseconds": "a span in hundredths of a second (sysUpTime)",
"bytes": "a size in bytes",
"kilobytes": "a size in KiB",
"octets": "a byte counter, shown as traffic (bit/s)",
"count": "any other counter, shown per second",
}
_UPTIME = Reading("1.3.6.1.2.1.1.3.0", "uptime", "Uptime", "centiseconds")
# id: (label, what it is for, readings). Interface traffic is per interface
# and built by interface_readings from the numbers on the form.
PRESETS: dict[str, tuple[str, str, tuple[Reading, ...]]] = {
"system": ("System", "How long the device has been up. Nearly every SNMP device answers it.",
(_UPTIME,)),
"ups": ("UPS", "Battery state, charge, runtime left, load and input voltage (UPS-MIB, RFC 1628; "
"APC, Eaton and CyberPower management cards).",
(Reading("1.3.6.1.2.1.33.1.2.1.0", "battery_status", "Battery", "ups_battery"),
Reading("1.3.6.1.2.1.33.1.2.4.0", "battery_charge", "Charge", "percent"),
Reading("1.3.6.1.2.1.33.1.2.3.0", "runtime_left", "Runtime left", "minutes"),
Reading("1.3.6.1.2.1.33.1.4.4.1.5.1", "load", "Load", "percent"),
Reading("1.3.6.1.2.1.33.1.3.3.1.3.1", "input_voltage", "Input", "volts"),
Reading("1.3.6.1.2.1.33.1.2.2.0", "on_battery_for", "On battery", "battery_seconds"))),
"linux": ("Linux server", "CPU, memory and load from net-snmp (snmpd's UCD-SNMP-MIB).",
(Reading("1.3.6.1.4.1.2021.11.11.0", "cpu_used", "CPU used", "percent", scale=-1.0, offset=100.0),
Reading("1.3.6.1.4.1.2021.10.1.5.1", "load_1m", "Load (1m)", "number", scale=0.01),
Reading("1.3.6.1.4.1.2021.4.6.0", "memory_free", "Memory free", "kilobytes"),
Reading("1.3.6.1.4.1.2021.4.5.0", "memory_total", "Memory total", "kilobytes"))),
}
PRESET_ORDER = ("system", "interfaces", "ups", "linux")
INTERFACES_LABEL = ("Interface traffic", "Traffic in and out and link state for the interfaces numbered "
"below. Test the device to see its interfaces and their numbers.")
def parse_interfaces(text: str) -> list[int]:
"""'1, 2, 5-7' → [1, 2, 5, 6, 7]. Raises ValueError naming what it can't read."""
out: list[int] = []
for part in re.split(r"[,\s]+", (text or "").strip()):
if not part:
continue
m = re.fullmatch(r"(\d+)(?:-(\d+))?", part)
if not m:
raise ValueError(f"“{part}” is not an interface number or range")
lo, hi = int(m.group(1)), int(m.group(2) or m.group(1))
if hi < lo or hi - lo > 64:
raise ValueError(f"“{part}” is not a range of up to 64 interfaces")
out.extend(n for n in range(lo, hi + 1) if n not in out)
return out
def interface_readings(numbers: list[int], version: str) -> list[Reading]:
"""In, out and link state per interface. v2c and v3 read the 64-bit
counters (ifHCInOctets), which don't wrap on a fast link; v1 has only the
32-bit ones (ifInOctets)."""
hc = version != "1"
out: list[Reading] = []
for n in numbers:
rx = f"1.3.6.1.2.1.31.1.1.1.6.{n}" if hc else f"1.3.6.1.2.1.2.2.1.10.{n}"
tx = f"1.3.6.1.2.1.31.1.1.1.10.{n}" if hc else f"1.3.6.1.2.1.2.2.1.16.{n}"
out += [Reading(f"1.3.6.1.2.1.2.2.1.8.{n}", f"if{n}_status", f"Interface {n}", "if_status"),
Reading(rx, f"if{n}_in", f"Interface {n} in", "octets"),
Reading(tx, f"if{n}_out", f"Interface {n} out", "octets")]
return out
_OID = re.compile(r"\.?\d+(\.\d+)+")
_KEY = re.compile(r"[a-z][a-z0-9_]{0,63}")
def parse_custom(text: str) -> list[Reading]:
"""One reading a line: `OID name [unit]`. The name is a single word
(letters, digits, underscores) and becomes the metric name; the unit is
one of UNITS and defaults to number. Raises ValueError naming the line."""
out: list[Reading] = []
for i, raw in enumerate((text or "").splitlines(), start=1):
line = raw.split("#", 1)[0].strip()
if not line:
continue
parts = line.split()
if len(parts) not in (2, 3):
raise ValueError(f"Line {i}: write it as “OID name” or “OID name unit”")
oid, key = parts[0].lstrip("."), parts[1].lower()
unit = parts[2].lower() if len(parts) == 3 else "number"
if not _OID.fullmatch(oid):
raise ValueError(f"Line {i}: “{parts[0]}” is not an OID (numbers separated by dots)")
if not _KEY.fullmatch(key):
raise ValueError(f"Line {i}: the name “{parts[1]}” must start with a letter and use "
"only letters, digits and underscores")
if unit not in UNITS:
raise ValueError(f"Line {i}: “{parts[2]}” is not a unit; use one of {', '.join(UNITS)}")
out.append(Reading(oid, key, key.replace("_", " ").capitalize(), unit))
return out
def device_readings(presets: list[str], interfaces: list[int], custom: str, version: str) -> list[Reading]:
"""Every reading a device is asked for, presets in their order then custom
lines. Raises ValueError when two readings share a name."""
out: list[Reading] = []
for pid in PRESET_ORDER:
if pid == "interfaces":
if "interfaces" in presets:
out += interface_readings(interfaces, version)
elif pid in presets:
out += PRESETS[pid][2]
out += parse_custom(custom)
seen: set[str] = set()
for r in out:
if r.key in seen:
raise ValueError(f"Two readings are named “{r.key}”; give the custom one another name")
seen.add(r.key)
return out
def stored_unit(reading: Reading) -> str:
"""The unit a reading is stored in: a counter's rate unit, else its own."""
return COUNTER_UNITS[reading.unit][0] if reading.counter else reading.unit
def _bits(bps: float) -> str:
for unit, size in (("Gbit/s", 1e9), ("Mbit/s", 1e6), ("kbit/s", 1e3)):
if abs(bps) >= size:
return f"{bps / size:.3g} {unit}"
return f"{bps:.0f} bit/s"
def format_value(unit: str, value: float | None) -> str:
"""A stored value as a person reads it, in its unit."""
if value is None:
return "—"
if unit in STATUS_WORDS:
return STATUS_WORDS[unit].get(int(value), f"code {value:.0f}")
if unit == "centiseconds":
return human_duration(value / 100)
if unit == "seconds":
return human_duration(value)
if unit == "battery_seconds":
# UPS-MIB upsSecondsOnBattery: zero whenever the UPS is on mains power.
return f"for {human_duration(value)}" if value > 0 else "No, on mains"
if unit == "minutes":
return human_duration(value * 60)
if unit == "percent":
return f"{value:.0f}%"
if unit == "celsius":
return f"{value:.1f} °C"
if unit == "volts":
return f"{value:.0f} V"
if unit == "decivolts":
return f"{value / 10:.1f} V"
if unit == "bytes":
return human_bytes(value)
if unit == "kilobytes":
return human_bytes(value * 1024)
if unit == "bits_per_second":
return _bits(value)
if unit == "per_second":
return f"{value:.3g}/s"
return f"{value:,.0f}" if float(value).is_integer() else f"{value:.4g}"
# What a chart's axis says for a stored unit, and how its values scale for it.
def chart_axis(unit: str) -> tuple[str, float]:
return {"centiseconds": ("days", 1 / 8_640_000), "seconds": ("seconds", 1.0),
"battery_seconds": ("seconds on battery", 1.0),
"minutes": ("minutes", 1.0), "percent": ("%", 1.0), "celsius": ("°C", 1.0),
"volts": ("V", 1.0), "decivolts": ("V", 0.1), "bytes": ("MiB", 1 / 1_048_576),
"kilobytes": ("MiB", 1 / 1024), "bits_per_second": ("Mbit/s", 1e-6),
"per_second": ("per second", 1.0)}.get(unit, ("", 1.0))
plugins/snmp/routes.py
+179 -283
View File
@@ -1,319 +1,215 @@
# plugins/snmp/routes.py
"""The SNMP pages. Every GET reads Steward's own records: the device rows the
editor writes and the poller annotates, and the readings it stores. Only Test,
an admin's explicit action, asks a device anything (System 178's ruling)."""
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from quart import Blueprint, abort, current_app, redirect, render_template, request, session
from sqlalchemy import delete, select, update
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import func, select
from steward.auth.middleware import require_role
from steward.core.audit import log_audit
from steward.core.metric_series import latest_by_resource
from steward.models.metrics import PluginMetric, PluginMetricLatest
from steward.models.users import UserRole
from .models import SnmpDevice
from .readings import (INTERFACES_LABEL, PRESETS, PRESET_ORDER, UNITS, chart_axis, format_value,
stored_unit)
from .service import (FormError, device_state, form_credentials, form_values, parse_form,
readings_for)
from steward.models.metrics import PluginMetric
snmp_bp = Blueprint("snmp", __name__, template_folder="templates")
def _poll_seconds() -> int:
return int(current_app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60))
async def _latest_readings(db, device_names: list[str]) -> dict[str, dict[str, float]]:
"""Return {device_name: {label: latest_value}} for all configured devices."""
if not device_names:
return {}
# Latest value per (resource_name, metric_name) pair
subq = (
select(
PluginMetric.resource_name,
PluginMetric.metric_name,
func.max(PluginMetric.recorded_at).label("latest_at"),
)
.where(PluginMetric.source_module == "snmp")
.where(PluginMetric.resource_name.in_(device_names))
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
.subquery()
)
result = await db.execute(
select(PluginMetric).join(
subq,
(PluginMetric.resource_name == subq.c.resource_name)
& (PluginMetric.metric_name == subq.c.metric_name)
& (PluginMetric.recorded_at == subq.c.latest_at),
).where(PluginMetric.source_module == "snmp")
)
rows = result.scalars().all()
out: dict[str, dict[str, float]] = {}
for row in rows:
out.setdefault(row.resource_name, {})[row.metric_name] = row.value
return out
async def _all_devices(db) -> list[SnmpDevice]:
return list((await db.execute(select(SnmpDevice).order_by(SnmpDevice.name))).scalars().all())
async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]:
"""Return {label: [(recorded_at, value), ...]} for a single device."""
since = datetime.now(timezone.utc) - timedelta(hours=hours)
result = await db.execute(
select(PluginMetric)
.where(
PluginMetric.source_module == "snmp",
PluginMetric.resource_name == device_name,
PluginMetric.recorded_at >= since,
)
.order_by(PluginMetric.recorded_at)
)
rows = result.scalars().all()
async def _latest(db, names: list[str]) -> dict[str, dict]:
"""{device name: {key: value}} from plugin_metric_latest."""
latest = await latest_by_resource(db, "snmp", names)
return {name: {m: r.value for m, r in metrics.items()} for name, metrics in latest.items()}
def shown_readings(device: SnmpDevice, values: dict) -> list[dict]:
"""The device's readings in its own order, each named and in its unit;
"—" for one with no value yet."""
return [{"key": r.key, "name": r.name, "counter": r.counter,
"text": format_value(stored_unit(r), values.get(r.key))}
for r in readings_for(device)]
def view(device: SnmpDevice, values: dict, now: datetime) -> dict:
state = device_state(device, _poll_seconds(), now)
names = {r.key: r.name for r in readings_for(device)}
return {"id": device.id, "name": device.name, "address": device.address, "port": device.port,
"version": device.version, "reach": state.reach, "says": state.says,
"polled_at": device.last_polled_at, "ok_at": device.last_ok_at,
"readings": shown_readings(device, values),
"missing": [names.get(k, k) for k in device.missing or []]}
_REACH_ORDER = {"down": 0, "stale": 1, "pending": 2, "up": 3}
out: dict[str, list] = {}
for row in rows:
out.setdefault(row.metric_name, []).append(
(row.recorded_at.isoformat(), row.value)
)
return out
@snmp_bp.get("/")
@require_role(UserRole.viewer)
async def index():
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
async with current_app.db_sessionmaker() as db:
devices = await _all_devices(db)
latest = await _latest(db, [d.name for d in devices])
now = datetime.now(timezone.utc)
shown = [view(d, latest.get(d.name, {}), now) for d in devices]
shown.sort(key=lambda v: (_REACH_ORDER[v["reach"]], v["name"].lower()))
return await render_template("snmp/index.html", devices=shown)
latest = await _latest_readings(db, device_names)
# Merge config + latest readings for the template
devices = []
for cfg in devices_cfg:
name = cfg.get("name") or cfg.get("host", "?")
devices.append({
"name": name,
"host": cfg.get("host", ""),
"oids": cfg.get("oids", []),
"readings": latest.get(name, {}),
})
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
return await render_template("snmp/index.html",
devices=devices,
poll_interval=poll_interval)
# ── The editor ────────────────────────────────────────────────────────────────
def _device_binds_to_host(d: dict, keys: set[str], host_id: str) -> bool:
"""Does config device `d` belong on the host identified by `keys`
(lowercased {address, name}) / `host_id`?
async def _hosts(db) -> list:
from steward.models.hosts import Host
return list((await db.execute(select(Host.id, Host.name).order_by(Host.name))).all())
A device may declare an **explicit** binding that decouples "which Steward
host this belongs to" from "where to poll" (the `host` field):
• `host_id` — the Steward Host UUID (exact match), the explicit link.
• `steward_host` — a friendly bind by Host name or address (case-insensitive).
An explicit binding is **exclusive**: a device bound to host A must not also
match host B by a coincidental poll-target string. Only when no explicit
binding is present do we fall back to the implicit `host`-string match
(backward compatible with configs that only set `host`).
"""
explicit_id = str(d.get("host_id", "")).strip()
explicit_name = str(d.get("steward_host", "")).strip().lower()
if explicit_id or explicit_name:
if explicit_id and host_id and explicit_id == host_id:
return True
return bool(explicit_name and explicit_name in keys)
return str(d.get("host", "")).strip().lower() in keys
async def _render_form(device: SnmpDevice | None, values: dict, error: str | None = None, status: int = 200):
async with current_app.db_sessionmaker() as db:
hosts = await _hosts(db)
presets = [(pid, *(INTERFACES_LABEL if pid == "interfaces" else PRESETS[pid][:2])) for pid in PRESET_ORDER]
return await render_template("snmp/form.html", device=device, v=values, error=error, hosts=hosts,
presets=presets, units=UNITS), status
def _actor() -> tuple[str | None, str]:
return session.get("user_id"), session.get("username", "")
@snmp_bp.get("/devices/new")
@require_role(UserRole.admin)
async def new_device():
return await _render_form(None, form_values(None))
@snmp_bp.post("/devices/new")
@require_role(UserRole.admin)
async def create_device():
form = await request.form
try:
values = parse_form(form)
except FormError as exc:
return await _render_form(None, _typed(form), str(exc), 422)
async with current_app.db_sessionmaker() as db:
async with db.begin():
if await _name_taken(db, values["name"]):
return await _render_form(None, _typed(form), f"A device is already named “{values['name']}”.", 422)
device = SnmpDevice(**values)
db.add(device)
await db.flush()
device_id = device.id
await log_audit(current_app, *_actor(), "snmp.device_created", entity_type="snmp_device",
entity_id=device_id, detail={"name": values["name"], "address": values["address"]})
return redirect(f"/plugins/snmp/devices/{device_id}")
async def _name_taken(db, name: str, other_than: str | None = None) -> bool:
q = select(SnmpDevice.id).where(SnmpDevice.name == name)
if other_than:
q = q.where(SnmpDevice.id != other_than)
return (await db.execute(q)).first() is not None
def _typed(form) -> dict:
"""What was typed, to show the form again with an error. Secrets are not
echoed back."""
return {"name": form.get("name", ""), "address": form.get("address", ""), "port": form.get("port", ""),
"version": form.get("version", "2c"), "v3_user": form.get("v3_user", ""),
"v3_auth": form.get("v3_auth", "none"), "v3_priv": form.get("v3_priv", "none"),
"presets": form.getlist("presets"), "interfaces": form.get("interfaces", ""),
"custom": form.get("custom", ""), "host_id": form.get("host_id", "")}
async def _device_or_404(db, device_id: str) -> SnmpDevice:
device = await db.get(SnmpDevice, device_id)
if device is None:
abort(404)
return device
@snmp_bp.get("/devices/<device_id>/edit")
@require_role(UserRole.admin)
async def edit_device(device_id: str):
async with current_app.db_sessionmaker() as db:
device = await _device_or_404(db, device_id)
return await _render_form(device, form_values(device))
@snmp_bp.post("/devices/<device_id>/edit")
@require_role(UserRole.admin)
async def save_device(device_id: str):
form = await request.form
async with current_app.db_sessionmaker() as db:
async with db.begin():
device = await _device_or_404(db, device_id)
try:
values = parse_form(form, device)
except FormError as exc:
return await _render_form(device, {**_typed(form), **_kept_secrets(device)}, str(exc), 422)
if await _name_taken(db, values["name"], other_than=device.id):
return await _render_form(device, {**_typed(form), **_kept_secrets(device)},
f"A device is already named “{values['name']}”.", 422)
old_name = device.name
for column, value in values.items():
setattr(device, column, value)
# What it is asked, or how, may have changed: the last outcome no
# longer describes it, so the page says "not polled yet" until the next poll.
device.last_polled_at = device.last_ok_at = device.last_error = None
device.missing = []
if old_name != device.name:
# Its history is stored under its name; move it with the rename.
for model in (PluginMetric, PluginMetricLatest):
await db.execute(update(model).where(
model.source_module == "snmp", model.resource_name == old_name
).values(resource_name=device.name))
from . import scheduler
if scheduler.RATES is not None:
scheduler.RATES.forget(device_id)
detail = {"name": values["name"]}
if old_name != values["name"]:
detail["old_name"] = old_name
await log_audit(current_app, *_actor(), "snmp.device_updated", entity_type="snmp_device",
entity_id=device_id, detail=detail)
return redirect(f"/plugins/snmp/devices/{device_id}")
def _kept_secrets(device: SnmpDevice) -> dict:
return {"has_community": bool(device.community), "has_auth_key": bool(device.v3_auth_key),
"has_priv_key": bool(device.v3_priv_key)}
@snmp_bp.post("/devices/<device_id>/delete")
@require_role(UserRole.admin)
async def delete_device(device_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
device = await _device_or_404(db, device_id)
name = device.name
await db.delete(device)
# Its newest readings go with it, so nothing reads it as a device
# that stopped answering. Its history ages out with retention.
await db.execute(delete(PluginMetricLatest).where(
PluginMetricLatest.source_module == "snmp", PluginMetricLatest.resource_name == name))
await log_audit(current_app, *_actor(), "snmp.device_deleted", entity_type="snmp_device",
entity_id=device_id, detail={"name": name})
return redirect("/plugins/snmp/")
@snmp_bp.post("/devices/test")
@require_role(UserRole.admin)
async def test_device():
"""Ask the device on the (unsaved) form who it is and what interfaces it
has. An admin's explicit action, so it may contact the device."""
from .poller import describe_device
form = await request.form
existing = None
if form.get("device_id"):
async with current_app.db_sessionmaker() as db:
existing = await db.get(SnmpDevice, form["device_id"])
address, port, creds = form_credentials(form, existing)
if not address:
found = {"error": "Enter the device's address first."}
else:
found = await describe_device(address, port, creds)
return await render_template("snmp/_test.html", found=found, address=address, port=port)
# ── The device page, the dashboard widget, the host page's panel ─────────────
async def _history(db, device_name: str, since: datetime) -> dict[str, list]:
"""{key: [(iso time, value), ...]} for one device since `since`."""
rows = (await db.execute(
select(PluginMetric.metric_name, PluginMetric.recorded_at, PluginMetric.value)
.where(PluginMetric.source_module == "snmp", PluginMetric.resource_name == device_name,
PluginMetric.recorded_at >= since)
.order_by(PluginMetric.recorded_at))).all()
out: dict[str, list] = {}
for key, at, value in rows:
out.setdefault(key, []).append((at.isoformat(), value))
return out
RANGES = (1, 6, 24, 168)
@snmp_bp.get("/devices/<device_id>")
@require_role(UserRole.viewer)
async def device_detail(device_id: str):
hours = request.args.get("hours", type=int, default=24)
if hours not in RANGES:
hours = 24
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
device = await _device_or_404(db, device_id)
latest = await _latest(db, [device.name])
hist = await _history(db, device.name, now - timedelta(hours=hours))
charts = []
for r in readings_for(device):
unit = stored_unit(r)
axis, factor = chart_axis(unit)
charts.append({"key": r.key, "name": r.name, "axis": axis,
"now": format_value(unit, latest.get(device.name, {}).get(r.key)),
"points": [(t, v * factor) for t, v in hist.get(r.key, [])]})
return await render_template("snmp/device.html", device=view(device, latest.get(device.name, {}), now),
charts=charts, charts_json=json.dumps(charts), hours=hours, ranges=RANGES)
DEVICES_SHOWN = 10
@snmp_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""The dashboard SNMP widget (#5255), in widget shape: a verdict, devices
not answering first with why, then the rest with two readings each."""
async with current_app.db_sessionmaker() as db:
devices = await _all_devices(db)
latest = await _latest(db, [d.name for d in devices])
now = datetime.now(timezone.utc)
shown = [view(d, latest.get(d.name, {}), now) for d in devices]
shown.sort(key=lambda v: (_REACH_ORDER[v["reach"]], v["name"].lower()))
for v in shown:
v["key"] = [r for r in v["readings"] if r["text"] != "—"][:2]
counts = {r: sum(1 for d in shown if d["reach"] == r) for r in _REACH_ORDER}
return await render_template("snmp/widget.html", devices=shown[:DEVICES_SHOWN],
more=max(0, len(shown) - DEVICES_SHOWN), counts=counts, total=len(shown))
def devices_for_host(devices: list[SnmpDevice], host_id: str, address: str | None,
name: str | None) -> list[SnmpDevice]:
"""The devices shown on a host's page: those bound to it, and those bound
to no host whose address is the host's address or name. A binding is
exclusive, so a device bound elsewhere never matches by address."""
keys = {(address or "").strip().lower(), (name or "").strip().lower()} - {""}
return [d for d in devices
if (d.host_id == host_id if d.host_id else d.address.strip().lower() in keys)]
def _devices_for_host(devices_cfg: list, address: str | None, name: str | None,
host_id: str | None = None) -> list[dict]:
"""SNMP devices that map onto a Steward host's page. Prefers an explicit
per-device binding (`host_id` / `steward_host`); otherwise falls back to
matching the device's `host` (poll target) against the host's address or
name (case-insensitive). See `_device_binds_to_host` for the precedence."""
keys = {(address or "").strip().lower(), (name or "").strip().lower()}
keys.discard("")
hid = (host_id or "").strip()
return [
d for d in devices_cfg
if isinstance(d, dict) and _device_binds_to_host(d, keys, hid)
]
@snmp_bp.get("/host/<host_id>")
@require_role(UserRole.viewer)
async def host_panel(host_id: str):
"""The SNMP panel on a host's page; nothing when no device maps to it."""
"""Per-host SNMP fragment for the Hosts hub. Surfaces any configured SNMP
device that maps to this host (by address or name) with its latest readings.
Renders nothing when no device maps, so hosts without SNMP carry no empty card.
"""
from steward.models.hosts import Host
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
async with current_app.db_sessionmaker() as db:
host = await db.get(Host, host_id)
host = (await db.execute(
select(Host).where(Host.id == host_id))).scalar_one_or_none()
if host is None:
return ""
matched = devices_for_host(await _all_devices(db), host.id, host.address, host.name)
matched = _devices_for_host(devices_cfg, host.address, host.name, host.id)
if not matched:
return ""
latest = await _latest(db, [d.name for d in matched])
now = datetime.now(timezone.utc)
return await render_template("snmp/host_panel.html",
devices=[{**view(d, latest.get(d.name, {}), now), "bound": bool(d.host_id)}
for d in matched])
names = [d.get("name") or d.get("host", "?") for d in matched]
latest = await _latest_readings(db, names)
devices = [{
"name": d.get("name") or d.get("host", "?"),
"host": d.get("host", ""),
"oids": d.get("oids", []),
"readings": latest.get(d.get("name") or d.get("host", "?"), {}),
"bound": bool(str(d.get("host_id", "")).strip()
or str(d.get("steward_host", "")).strip()),
} for d in matched]
return await render_template("snmp/host_panel.html", devices=devices)
@snmp_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
async with current_app.db_sessionmaker() as db:
latest = await _latest_readings(db, device_names[:10])
devices = []
for cfg in devices_cfg[:10]:
name = cfg.get("name") or cfg.get("host", "?")
devices.append({
"name": name,
"oids": cfg.get("oids", []),
"readings": latest.get(name, {}),
})
return await render_template("snmp/widget.html",
devices=devices,
total=len(devices_cfg))
@snmp_bp.get("/device/<device_name>")
@require_role(UserRole.viewer)
async def device_detail(device_name: str):
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
device_cfg = next(
(d for d in devices_cfg if (d.get("name") or d.get("host")) == device_name),
None,
)
if device_cfg is None:
from quart import abort
abort(404)
hours = int(request.args.get("hours", 24))
async with current_app.db_sessionmaker() as db:
hist = await _history(db, device_name, hours=hours)
import json
history_json = json.dumps(hist)
oid_labels = [o.get("label") or o.get("oid") for o in device_cfg.get("oids", [])]
return await render_template(
"snmp/device.html",
device=device_cfg,
device_name=device_name,
oid_labels=oid_labels,
history_json=history_json,
hours=hours,
)
plugins/snmp/scheduler.py
+35 -56
View File
@@ -1,22 +1,14 @@
# plugins/snmp/scheduler.py
from __future__ import annotations
import asyncio
import logging
from datetime import datetime, timezone
from typing import TYPE_CHECKING
from sqlalchemy import select
from steward.core.scheduler import ScheduledTask
if TYPE_CHECKING:
from quart import Quart
logger = logging.getLogger(__name__)
from steward.core.scheduler import ScheduledTask
# Counters' previous samples, for their rates (service.RateKeeper).
RATES = None
logger = logging.getLogger(__name__)
def make_poll_task(app: "Quart") -> ScheduledTask:
@@ -33,58 +25,45 @@ def make_poll_task(app: "Quart") -> ScheduledTask:
)
async def _do_poll(app: "Quart", poll=None) -> None:
"""Poll every device at once, then write each one's outcome to its row
and its answers to the metric tables. `poll` stands in for
poller.poll_device in tests."""
global RATES
from steward.core.alerts import MetricBatch
from .models import SnmpDevice
async def _do_poll(app: "Quart") -> None:
from .poller import poll_device
from .service import RateKeeper, credentials, readings_for
from steward.core.alerts import record_metric
poll = poll or poll_device
if RATES is None:
RATES = RateKeeper()
async with app.db_sessionmaker() as session:
devices = (await session.execute(select(SnmpDevice))).scalars().all()
devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", [])
if not devices:
return
asked = {d.id: readings_for(d) for d in devices}
results = await asyncio.gather(*(
poll(d.address, d.port, credentials(d), [(r.oid, r.key) for r in asked[d.id]])
for d in devices), return_exceptions=True)
now = datetime.now(timezone.utc)
async with app.db_sessionmaker() as session:
async with session.begin():
batch = MetricBatch()
for device, result in zip(devices, results):
row = await session.get(SnmpDevice, device.id)
if row is None: # deleted while it was being polled
for device in devices:
if not isinstance(device, dict):
continue
row.last_polled_at = now
if isinstance(result, BaseException):
logger.warning("SNMP poll of %s failed: %s", device.name, result)
row.last_error = f"SNMP poll failed: {result}"
name = device.get("name") or device.get("host", "unknown")
host = device.get("host", "")
port = int(device.get("port", 161))
community = device.get("community", "public")
version = str(device.get("version", "2c"))
oids = device.get("oids", [])
if not host or not oids:
continue
row.last_error = result.error
row.missing = sorted(set(result.missing))
if result.readings:
row.last_ok_at = now
for reading in asked[device.id]:
value = result.readings.get(reading.key)
if value is None:
continue
if reading.counter:
value = RATES.rate(device.id, reading, value, now)
if value is None:
continue
else:
value = value * reading.scale + reading.offset
batch.add(source_module="snmp", resource_name=device.name,
metric_name=reading.key, value=value, recorded_at=now)
await batch.record(session)
try:
readings = await poll_device(host, port, community, version, oids)
except Exception:
logger.exception("SNMP poll failed for device %s (%s)", name, host)
continue
for label, value in readings.items():
await record_metric(
session=session,
source_module="snmp",
resource_name=name,
metric_name=label,
value=value,
)
if readings:
logger.debug("SNMP polled %s (%s): %d OID(s)", name, host, len(readings))
else:
logger.debug("SNMP polled %s (%s): no readings", name, host)
plugins/snmp/service.py
-195
View File
@@ -1,195 +0,0 @@
# plugins/snmp/service.py
"""SNMP devices: the form a person edits, the row it becomes, what the poller
asks for, and what a page says about the newest poll."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from steward.core.crypto import decrypt_secret, encrypt_secret
from .models import SnmpDevice
from .poller import AUTH_PROTOCOLS, PRIV_PROTOCOLS, Credentials
from .readings import PRESETS, Reading, device_readings, parse_interfaces
VERSIONS = ("1", "2c", "3")
PRESET_IDS = set(PRESETS) | {"interfaces"}
SECRET_FIELDS = ("community", "v3_auth_key", "v3_priv_key")
class FormError(ValueError):
"""What is wrong with a submitted device form, said to the person."""
def readings_for(device: SnmpDevice) -> list[Reading]:
"""What the device is asked for. A row saved through the form always
parses; one that somehow doesn't is asked for nothing rather than raising
in the poll loop."""
try:
return device_readings(device.presets or [], parse_interfaces(device.interfaces),
device.custom, device.version)
except ValueError:
return []
def credentials(device: SnmpDevice) -> Credentials:
return Credentials(
version=device.version,
community=decrypt_secret(device.community, context=f"snmp device {device.name} community"),
user=device.v3_user, auth=device.v3_auth,
auth_key=decrypt_secret(device.v3_auth_key, context=f"snmp device {device.name} auth key"),
priv=device.v3_priv,
priv_key=decrypt_secret(device.v3_priv_key, context=f"snmp device {device.name} privacy key"))
def form_values(device: SnmpDevice | None) -> dict:
"""The form's fields for an existing device. Secrets are never sent back
to the browser; a blank secret field on save keeps the stored one."""
if device is None:
return {"port": 161, "version": "2c", "presets": ["system"], "v3_auth": "sha", "v3_priv": "aes128"}
return {"name": device.name, "address": device.address, "port": device.port,
"version": device.version, "v3_user": device.v3_user, "v3_auth": device.v3_auth,
"v3_priv": device.v3_priv, "presets": list(device.presets or []),
"interfaces": device.interfaces, "custom": device.custom, "host_id": device.host_id or "",
"has_community": bool(device.community), "has_auth_key": bool(device.v3_auth_key),
"has_priv_key": bool(device.v3_priv_key)}
def parse_form(form, existing: SnmpDevice | None = None) -> dict:
"""The submitted form as the columns to store, secrets encrypted.
Raises FormError saying what to fix."""
name = (form.get("name") or "").strip()
address = (form.get("address") or "").strip()
if not name:
raise FormError("Give the device a name.")
if len(name) > 100 or "/" in name:
raise FormError("A name is up to 100 characters, without a slash.")
if not address:
raise FormError("Give the device's address: an IP or a hostname.")
try:
port = int(form.get("port") or 161)
except ValueError:
raise FormError("The port is a number, usually 161.") from None
if not 1 <= port <= 65535:
raise FormError("The port is between 1 and 65535, usually 161.")
version = form.get("version") or "2c"
if version not in VERSIONS:
raise FormError("Choose SNMP version 1, 2c or 3.")
presets = [p for p in form.getlist("presets") if p in PRESET_IDS]
interfaces = (form.get("interfaces") or "").strip()
custom = (form.get("custom") or "").strip()
try:
numbers = parse_interfaces(interfaces)
if "interfaces" in presets and not numbers:
raise FormError("Interface traffic needs the interface numbers to read, e.g. 1, 2 or 1-4.")
if not device_readings(presets, numbers, custom, version):
raise FormError("Choose at least one preset or add a custom reading.")
except ValueError as exc:
raise FormError(str(exc)) from None
def secret(field: str) -> str:
typed = form.get(field) or ""
if typed:
return encrypt_secret(typed)
return getattr(existing, field) if existing is not None else ""
values = {"name": name, "address": address, "port": port, "version": version,
"presets": presets, "interfaces": interfaces, "custom": custom,
"host_id": (form.get("host_id") or "").strip() or None,
"community": "", "v3_user": "", "v3_auth": "none", "v3_auth_key": "",
"v3_priv": "none", "v3_priv_key": ""}
if version == "3":
user = (form.get("v3_user") or "").strip()
auth = form.get("v3_auth") or "none"
priv = form.get("v3_priv") or "none"
if not user:
raise FormError("SNMPv3 needs a user name.")
if auth not in AUTH_PROTOCOLS or priv not in PRIV_PROTOCOLS:
raise FormError("Choose an authentication and a privacy protocol from the lists.")
if priv != "none" and auth == "none":
raise FormError("Privacy needs authentication too: choose an authentication protocol.")
values.update(v3_user=user, v3_auth=auth, v3_priv=priv,
v3_auth_key=secret("v3_auth_key") if auth != "none" else "",
v3_priv_key=secret("v3_priv_key") if priv != "none" else "")
for field, proto in (("v3_auth_key", auth), ("v3_priv_key", priv)):
if proto != "none" and not values[field]:
raise FormError("SNMPv3 with " + ("authentication" if field == "v3_auth_key" else "privacy")
+ " needs its key (at least 8 characters).")
if proto != "none" and form.get(field) and len(form.get(field)) < 8:
raise FormError("An SNMPv3 key is at least 8 characters.")
else:
values["community"] = secret("community") or encrypt_secret("public")
return values
def form_credentials(form, existing: SnmpDevice | None = None) -> tuple[str, int, Credentials]:
"""Address, port and credentials from an unsaved form, for Test. A blank
secret falls back to the stored one when testing an existing device."""
def secret(field: str, default: str = "") -> str:
typed = form.get(field) or ""
if typed:
return typed
if existing is not None and getattr(existing, field):
return decrypt_secret(getattr(existing, field))
return default
version = form.get("version") if form.get("version") in VERSIONS else "2c"
try:
port = int(form.get("port") or 161)
except ValueError:
port = 161
creds = Credentials(version=version, community=secret("community", "public"),
user=(form.get("v3_user") or "").strip(), auth=form.get("v3_auth") or "none",
auth_key=secret("v3_auth_key"), priv=form.get("v3_priv") or "none",
priv_key=secret("v3_priv_key"))
return (form.get("address") or "").strip(), port, creds
# A device not polled for this many intervals has stopped being polled
# (plugin turned off, scheduler stuck), whatever its last outcome said.
STALE_POLLS = 3
@dataclass
class DeviceState:
reach: str # up | down | stale | pending
says: str # one line, in words
def device_state(device: SnmpDevice, poll_seconds: int, now: datetime) -> DeviceState:
"""What the newest poll says about the device."""
if device.last_polled_at is None:
return DeviceState("pending", "Not polled yet.")
if (now - device.last_polled_at).total_seconds() > STALE_POLLS * max(int(poll_seconds), 1):
return DeviceState("stale", "Not polled recently: the SNMP plugin may be turned off.")
if device.last_error:
return DeviceState("down", device.last_error)
return DeviceState("up", "Answering.")
class RateKeeper:
"""A counter's previous sample per (device, key), to turn the next one
into a per-second rate. In memory: after a restart the first poll of each
counter stores nothing and the second stores its rate. A sample lower
than the one before (the device restarted, or a 32-bit counter wrapped)
is skipped rather than read as a huge negative or wrapped rate."""
def __init__(self) -> None:
self._last: dict[tuple[str, str], tuple[float, datetime]] = {}
def rate(self, device_id: str, reading: Reading, value: float, at: datetime) -> float | None:
from .readings import COUNTER_UNITS
key = (device_id, reading.key)
prev = self._last.get(key)
self._last[key] = (value, at)
if prev is None:
return None
seconds = (at - prev[1]).total_seconds()
if seconds <= 0 or value < prev[0]:
return None
return (value - prev[0]) / seconds * COUNTER_UNITS[reading.unit][1]
def forget(self, device_id: str) -> None:
for key in [k for k in self._last if k[0] == device_id]:
del self._last[key]
plugins/snmp/templates/snmp/_test.html
-22
View File
@@ -1,22 +0,0 @@
{# What Test found (milestone 491): who the device says it is and its
interfaces with their numbers, or why it didn't answer. #}
{% if found.error %}
<p class="tone-crit" role="alert" style="margin:0.75rem 0 0;">{{ found.error }}</p>
{% else %}
<div style="margin-top:0.75rem;">
<p class="tone-ok" style="margin:0 0 0.4rem;">{{ address }} answered.</p>
<dl class="test-facts" style="display:grid;grid-template-columns:max-content 1fr;gap:0.2rem 0.9rem;margin:0;font-size:0.85rem;">
<dt class="tone-dim">Name</dt><dd style="margin:0;">{{ found.name or "—" }}</dd>
<dt class="tone-dim">Description</dt><dd style="margin:0;overflow-wrap:anywhere;">{{ found.description or "—" }}</dd>
<dt class="tone-dim">Up for</dt><dd style="margin:0;">{{ (found.uptime_cs / 100)|duration if found.uptime_cs is not none else "—" }}</dd>
</dl>
{% if found.interfaces %}
<p class="field-hint" style="margin:0.6rem 0 0.3rem;">Interfaces, by the number to put in Interface traffic:</p>
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(11rem,1fr));gap:0.15rem 0.9rem;font-size:0.82rem;">
{% for number, name in found.interfaces %}<span><span class="mono tone-dim">{{ number }}</span> {{ name }}</span>{% endfor %}
</div>
{% else %}
<p class="field-hint" style="margin:0.6rem 0 0;">It lists no interfaces.</p>
{% endif %}
</div>
{% endif %}
plugins/snmp/templates/snmp/device.html
+73 -71
View File
@@ -1,83 +1,85 @@
{# plugins/snmp/templates/snmp/device.html — one SNMP device (milestone 491):
what its newest poll found, then a small chart per reading in that
reading's own unit, on a dated axis. #}
{# plugins/snmp/templates/snmp/device.html #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header, status_badge %}
{% block title %}SNMP — {{ device.name }} — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("SNMP", "/plugins/snmp/"), (device.name, "")]) }}{% endblock %}
{% from "_macros.html" import crumbs %}
{% block title %}SNMP — {{ device_name }} — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("SNMP", "/plugins/snmp/"), (device_name, "")]) }}{% endblock %}
{% block content %}
{% call page_header(device.name, device.address ~ ((":" ~ device.port) if device.port != 161 else "") ~ " · SNMP v" ~ device.version) %}
{% for h in ranges %}
<a href="?hours={{ h }}" class="range-btn{% if hours == h %} active{% endif %}"{% if hours == h %} aria-current="true"{% endif %}>{% if h < 168 %}{{ h }}h{% else %}7d{% endif %}</a>
{% endfor %}
{% if session.user_role == 'admin' %}<a href="/plugins/snmp/devices/{{ device.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>{% endif %}
{% endcall %}
<div style="display:flex;align-items:center;gap:1rem;margin-bottom:1.5rem;flex-wrap:wrap;">
<h1 class="page-title" style="margin:0;">{{ device_name }}</h1>
<span style="font-size:0.8rem;color:var(--text-dim);">{{ device.host }}</span>
<div style="margin-left:auto;display:flex;gap:0.5rem;">
{% for h in [1, 6, 24, 168] %}
<a href="?hours={{ h }}"
class="btn btn-ghost"
style="font-size:0.78rem;padding:0.2rem 0.6rem;{% if hours == h %}background:var(--accent);color:#fff;border-color:var(--accent);{% endif %}">
{% if h < 24 %}{{ h }}h{% elif h == 24 %}24h{% else %}7d{% endif %}
</a>
{% endfor %}
</div>
</div>
<p class="page-verdict">
{% if device.reach == 'up' %}{{ status_badge('up', 'Answering') }} polled {{ device.polled_at|ago }}
{% elif device.reach == 'down' %}{{ status_badge('down', 'Not answering') }} <span class="tone-crit">{{ device.says }}</span>
{% if device.ok_at %}Last answered {{ device.ok_at|ago }}.{% else %}It has never answered.{% endif %}
{% elif device.reach == 'stale' %}{{ status_badge('stale', 'Not polled') }} {{ device.says }}
{% else %}{{ status_badge('pending', 'Not polled yet') }} The first poll runs within a minute of saving.{% endif %}
</p>
{% if device.missing %}<p class="field-hint">Not answered by this device: {{ device.missing|join(", ") }}.</p>{% endif %}
{% if charts %}
<div class="node-grid">
{% for c in charts %}
<section class="card" aria-labelledby="snmp-chart-{{ loop.index }}" style="margin:0;">
<div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.75rem;">
<h2 id="snmp-chart-{{ loop.index }}" class="section-title" style="margin:0;">{{ c.name }}</h2>
<span class="stat-tile-value" style="font-size:1rem;">{{ c.now }}</span>
</div>
<div style="position:relative;height:140px;margin-top:0.5rem;">
<canvas id="snmp-c-{{ loop.index0 }}" role="img" aria-label="{{ c.name }} over the last {{ hours }} hours"></canvas>
</div>
</section>
{% endfor %}
{% if oid_labels %}
<div class="card">
<canvas id="snmp-chart" style="width:100%;max-height:320px;"></canvas>
</div>
{% else %}
<div class="card field-hint">This device has no readings chosen{% if session.user_role == 'admin' %}; <a href="/plugins/snmp/devices/{{ device.id }}/edit">Edit</a> to choose some{% endif %}.</div>
{% endif %}
<script>
(function () {
var charts = {{ charts_json | safe }};
var hours = {{ hours }};
var line = stewardColor("--blue");
function stamp(iso) {
var d = new Date(iso);
return hours > 24
? d.toLocaleDateString([], { weekday: "short", day: "numeric" }) + " " + d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false })
: d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false });
(function(){
const raw = {{ history_json | safe }};
const labels = {{ oid_labels | tojson }};
const palette = ["#6060c0","#e07040","#40b080","#c040c0","#4080e0","#e0c040","#c08040"];
// Collect all unique timestamps, sorted
const tsSet = new Set();
for (const pts of Object.values(raw)) {
for (const [ts] of pts) tsSet.add(ts);
}
charts.forEach(function (c, i) {
var el = document.getElementById("snmp-c-" + i);
if (!el) return;
if (c.points.length < 2) {
el.parentElement.innerHTML = '<p class="field-hint" style="margin:0;">Not enough readings yet for a chart.</p>';
return;
}
new Chart(el.getContext("2d"), {
type: "line",
data: { labels: c.points.map(function (p) { return p[0]; }),
datasets: [{ data: c.points.map(function (p) { return p[1]; }), borderColor: line,
backgroundColor: "transparent", borderWidth: 1.5, pointRadius: 0, tension: 0.2 }] },
options: {
responsive: true, maintainAspectRatio: false, animation: false,
interaction: { mode: "index", intersect: false },
plugins: { legend: { display: false },
tooltip: { callbacks: { title: function (items) {
return items.length ? new Date(c.points[items[0].dataIndex][0]).toLocaleString([], { hour12: false }) : ""; } } } },
scales: {
x: { ticks: { maxTicksLimit: 6, maxRotation: 0, font: { size: 10 },
callback: function (v) { return stamp(c.points[v][0]); } }, grid: { display: false } },
y: { ticks: { font: { size: 10 }, maxTicksLimit: 4 },
title: { display: !!c.axis, text: c.axis, font: { size: 9 } } }
}
}
const allTs = Array.from(tsSet).sort();
if (allTs.length < 2) {
document.getElementById("snmp-chart").parentElement.innerHTML =
'<p style="color:var(--text-muted);padding:1rem;">Not enough data for a chart yet.</p>';
return;
}
const datasets = [];
labels.forEach((label, i) => {
const pts = raw[label] || [];
const map = Object.fromEntries(pts.map(([ts, v]) => [ts, v]));
datasets.push({
label,
data: allTs.map(ts => map[ts] ?? null),
borderColor: palette[i % palette.length],
backgroundColor: "transparent",
borderWidth: 1.5,
pointRadius: 0,
tension: 0.2,
spanGaps: true,
});
});
const ctx = document.getElementById("snmp-chart").getContext("2d");
new Chart(ctx, {
type: "line",
data: { labels: allTs.map(ts => ts.substring(11, 16)), datasets },
options: {
responsive: true,
interaction: { mode: "index", intersect: false },
plugins: {
legend: { position: "top", labels: { color: "#aaa", boxWidth: 12 } },
tooltip: { backgroundColor: "#1e1e2e", titleColor: "#ccc", bodyColor: "#aaa" },
},
scales: {
x: { ticks: { color: "#888", maxTicksLimit: 12 }, grid: { color: "#2a2a3a" } },
y: { ticks: { color: "#888" }, grid: { color: "#2a2a3a" } },
},
},
});
})();
</script>
{% else %}
<div class="card" style="color:var(--text-muted);text-align:center;padding:2rem;">
No OIDs configured for this device.
</div>
{% endif %}
{% endblock %}
plugins/snmp/templates/snmp/form.html
-166
View File
@@ -1,166 +0,0 @@
{# plugins/snmp/templates/snmp/form.html — add or edit an SNMP device
(milestone 491): where it is, how to sign in, what to read, and Test to
ask the device before saving. Secrets are never sent back to the browser;
a blank secret field keeps the stored one. #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header %}
{% block title %}{{ ("Edit " ~ device.name) if device else "Add SNMP device" }} — Steward{% endblock %}
{% block breadcrumb %}
{%- if device -%}
{{ crumbs([("SNMP", "/plugins/snmp/"), (device.name, "/plugins/snmp/devices/" ~ device.id), ("Edit", "")]) }}
{%- else -%}
{{ crumbs([("SNMP", "/plugins/snmp/"), ("Add device", "")]) }}
{%- endif -%}
{% endblock %}
{% block content %}
{{ page_header(("Edit " ~ device.name) if device else "Add SNMP device") }}
<div class="form-page">
{% if error %}<p class="field-error" role="alert">{{ error }}</p>{% endif %}
<form method="post" id="snmp-form" action="{{ '/plugins/snmp/devices/' ~ device.id ~ '/edit' if device else '/plugins/snmp/devices/new' }}">{{ csrf_field() }}
{% if device %}<input type="hidden" name="device_id" value="{{ device.id }}">{% endif %}
<section class="card form-section" aria-labelledby="snmp-where">
<h2 id="snmp-where" class="section-title">Device</h2>
<div class="form-pair">
<div class="field">
<label class="field-label" for="snmp-name">Name</label>
<input class="input" id="snmp-name" name="name" required maxlength="100" value="{{ v.name or '' }}" placeholder="core-switch" autocomplete="off">
</div>
<div class="field">
<label class="field-label" for="snmp-address">Address</label>
<input class="input mono" id="snmp-address" name="address" required value="{{ v.address or '' }}" placeholder="192.168.1.2 or switch.lan" autocomplete="off">
</div>
<div class="field">
<label class="field-label" for="snmp-port">Port</label>
<input class="input" id="snmp-port" name="port" type="number" min="1" max="65535" value="{{ v.port or 161 }}">
</div>
<div class="field">
<label class="field-label" for="snmp-host">Show on host</label>
<select class="input" id="snmp-host" name="host_id">
<option value="">The host with this address, if any</option>
{% for h in hosts %}<option value="{{ h.id }}"{% if v.host_id == h.id %} selected{% endif %}>{{ h.name }}</option>{% endfor %}
</select>
<span class="field-hint">Which host's page shows this device's readings.</span>
</div>
</div>
</section>
<section class="card form-section" aria-labelledby="snmp-auth">
<h2 id="snmp-auth" class="section-title">Sign-in</h2>
<div class="form-pair">
<div class="field">
<label class="field-label" for="snmp-version">SNMP version</label>
<select class="input" id="snmp-version" name="version">
{% for ver, words in [("2c", "v2c: a community string (most devices)"), ("3", "v3: a user with authentication and encryption"), ("1", "v1: a community string (old devices)")] %}
<option value="{{ ver }}"{% if v.version == ver %} selected{% endif %}>{{ words }}</option>
{% endfor %}
</select>
</div>
<div class="field" data-snmp="community">
<label class="field-label" for="snmp-community">Community</label>
<input class="input" id="snmp-community" name="community" type="password" autocomplete="new-password"
placeholder="{{ 'kept; type to change' if v.has_community else 'public' }}">
<span class="field-hint">The device's read community. Blank {{ "keeps the saved one" if v.has_community else "means public" }}.</span>
</div>
</div>
<div class="form-pair" data-snmp="v3">
<div class="field">
<label class="field-label" for="snmp-user">User</label>
<input class="input" id="snmp-user" name="v3_user" value="{{ v.v3_user or '' }}" autocomplete="off">
</div>
<div class="field"></div>
<div class="field">
<label class="field-label" for="snmp-auth-proto">Authentication</label>
<select class="input" id="snmp-auth-proto" name="v3_auth">
{% for val, words in [("sha", "SHA-1"), ("sha256", "SHA-256"), ("sha512", "SHA-512"), ("sha224", "SHA-224"), ("sha384", "SHA-384"), ("md5", "MD5"), ("none", "None")] %}
<option value="{{ val }}"{% if v.v3_auth == val %} selected{% endif %}>{{ words }}</option>
{% endfor %}
</select>
</div>
<div class="field">
<label class="field-label" for="snmp-auth-key">Authentication key</label>
<input class="input" id="snmp-auth-key" name="v3_auth_key" type="password" autocomplete="new-password"
placeholder="{{ 'kept; type to change' if v.has_auth_key else 'at least 8 characters' }}">
</div>
<div class="field">
<label class="field-label" for="snmp-priv-proto">Encryption</label>
<select class="input" id="snmp-priv-proto" name="v3_priv">
{% for val, words in [("aes128", "AES-128"), ("aes192", "AES-192"), ("aes256", "AES-256"), ("des", "DES"), ("3des", "3DES"), ("none", "None")] %}
<option value="{{ val }}"{% if v.v3_priv == val %} selected{% endif %}>{{ words }}</option>
{% endfor %}
</select>
</div>
<div class="field">
<label class="field-label" for="snmp-priv-key">Encryption key</label>
<input class="input" id="snmp-priv-key" name="v3_priv_key" type="password" autocomplete="new-password"
placeholder="{{ 'kept; type to change' if v.has_priv_key else 'at least 8 characters' }}">
</div>
</div>
<div style="display:flex;align-items:center;gap:0.75rem;flex-wrap:wrap;">
<button type="button" class="btn btn-sm btn-ghost"
hx-post="/plugins/snmp/devices/test" hx-include="#snmp-form" hx-target="#snmp-test"
hx-indicator="#snmp-testing">Test</button>
<span id="snmp-testing" class="field-hint htmx-indicator">Asking the device…</span>
<span class="field-hint">Asks the device for its name, uptime and interfaces, with what's typed here. Nothing is saved.</span>
</div>
<div id="snmp-test" aria-live="polite"></div>
</section>
<section class="card form-section" aria-labelledby="snmp-what">
<h2 id="snmp-what" class="section-title">Readings</h2>
<div class="check-list">
{% for pid, label, what in presets %}
<label style="display:flex;gap:0.5rem;align-items:flex-start;cursor:pointer;">
<input type="checkbox" name="presets" value="{{ pid }}"{% if pid in (v.presets or []) %} checked{% endif %} style="margin-top:0.2rem;">
<span><strong>{{ label }}</strong> <span class="field-hint">{{ what }}</span></span>
</label>
{% if pid == "interfaces" %}
<div class="field" style="margin:0 0 0.4rem 1.6rem;">
<label class="field-label" for="snmp-interfaces">Interface numbers</label>
<input class="input" id="snmp-interfaces" name="interfaces" value="{{ v.interfaces or '' }}" placeholder="1, 2, 5-8" autocomplete="off" style="max-width:20rem;">
</div>
{% endif %}
{% endfor %}
</div>
<div class="field" style="margin-top:0.9rem;">
<label class="field-label" for="snmp-custom">Custom readings</label>
<textarea class="input mono" id="snmp-custom" name="custom" rows="4" placeholder="1.3.6.1.4.1.318.1.1.1.2.2.2.0 battery_temperature celsius">{{ v.custom or '' }}</textarea>
<span class="field-hint">One a line: the OID, a name (letters, digits, underscores), and optionally a unit:
{% for u, words in units.items() %}<span class="mono">{{ u }}</span> ({{ words }}){{ ", " if not loop.last else "." }}{% endfor %}</span>
</div>
</section>
<div class="form-actions">
<button type="submit" class="btn">{{ "Save" if device else "Add device" }}</button>
<a href="{{ '/plugins/snmp/devices/' ~ device.id if device else '/plugins/snmp/' }}" class="btn btn-ghost">Cancel</a>
</div>
</form>
{% if device %}
<section class="card danger-zone" aria-labelledby="snmp-delete">
<div>
<h2 id="snmp-delete" class="section-title">Delete this device</h2>
<p class="field-hint">Steward stops polling it. Its past readings age out with the rest of the history.</p>
</div>
<form method="post" action="/plugins/snmp/devices/{{ device.id }}/delete"
data-confirm="Delete SNMP device {{ device.name }}? Steward stops polling it.">{{ csrf_field() }}
<button class="btn btn-danger" type="submit">Delete device</button>
</form>
</section>
{% endif %}
</div>
<script>
(function () {
var version = document.getElementById("snmp-version");
function show() {
var v3 = version.value === "3";
document.querySelectorAll('[data-snmp="v3"]').forEach(function (el) { el.hidden = !v3; });
document.querySelectorAll('[data-snmp="community"]').forEach(function (el) { el.hidden = v3; });
}
version.addEventListener("change", show);
show();
})();
</script>
{% endblock %}
plugins/snmp/templates/snmp/host_panel.html
+40 -17
View File
@@ -1,27 +1,50 @@
{# plugins/snmp/templates/snmp/host_panel.html — the SNMP panel on a host's
page: the devices bound to this host, or at its address, with their
readings in their units, or why they aren't answering. #}
{% from "_macros.html" import status_badge, stat_tile %}
{# Per-host SNMP fragment — embedded on the Hosts hub via HTMX. Self-contained.
Shows the SNMP device(s) whose address maps to this host + latest readings. #}
<div class="card">
<div style="display:flex;align-items:baseline;justify-content:space-between;margin-bottom:0.6rem;gap:1rem;flex-wrap:wrap;">
<h3 class="section-title" style="margin-bottom:0;">SNMP</h3>
<a href="/plugins/snmp/" class="field-hint">All devices</a>
<a href="/plugins/snmp/" style="font-size:0.8rem;color:var(--text-muted);">All devices →</a>
</div>
{% for d in devices %}
<div{% if not loop.last %} style="margin-bottom:1rem;"{% endif %}>
{% for device in devices %}
<div style="{% if not loop.last %}margin-bottom:1rem;{% endif %}">
<div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.5rem;flex-wrap:wrap;">
<a href="/plugins/snmp/devices/{{ d.id }}" style="font-weight:500;">{{ d.name }}</a>
<span class="field-hint mono">{{ d.address }}</span>
{% if d.reach == 'up' %}{{ status_badge('up', 'Answering') }}
{% elif d.reach == 'down' %}{{ status_badge('down', 'Not answering', title=d.says) }}
{% elif d.reach == 'stale' %}{{ status_badge('stale', 'Not polled', title=d.says) }}
{% else %}{{ status_badge('pending', 'Not polled yet') }}{% endif %}
<span style="font-weight:600;font-size:0.92rem;">{{ device.name }}</span>
<span style="font-size:0.78rem;color:var(--text-dim);font-family:ui-monospace,monospace;">{{ device.host }}</span>
{% if device.bound %}
<span title="Explicitly bound to this host (host_id / steward_host)"
style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:3px;padding:0.05rem 0.4rem;">bound</span>
{% endif %}
{% if device.readings %}
<span style="font-size:0.74rem;color:var(--green);">&#9679; reachable</span>
{% else %}
<span style="font-size:0.74rem;color:var(--text-dim);">&#9675; no data yet</span>
{% endif %}
<a href="/plugins/snmp/device/{{ device.name }}" class="btn btn-ghost btn-sm"
style="margin-left:auto;font-size:0.76rem;padding:0.15rem 0.55rem;">History</a>
</div>
{% if d.reach == 'down' %}<p class="field-hint" style="margin:0 0 0.5rem;">{{ d.says }}</p>{% endif %}
{% if d.reach != 'pending' and d.readings %}
<div class="stat-strip" style="margin-bottom:0;">
{% for r in d.readings %}{{ stat_tile(r.name, r.text) }}{% endfor %}
{% if device.readings %}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.5rem;">
{% for oid_cfg in device.oids %}
{% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
{% set val = device.readings.get(label) %}
<div style="background:var(--bg-elevated);border-radius:5px;padding:0.5rem 0.7rem;border:1px solid var(--border);">
<div style="font-size:0.68rem;color:var(--text-dim);margin-bottom:0.2rem;font-family:ui-monospace,monospace;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ label }}</div>
{% if val is not none %}
<div style="font-size:1rem;font-weight:600;font-variant-numeric:tabular-nums;">
{% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}<span style="font-size:0.72rem;color:var(--text-muted);">M</span>
{% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}<span style="font-size:0.72rem;color:var(--text-muted);">K</span>
{% else %}{{ "%.4g"|format(val) }}{% endif %}
</div>
{% else %}
<div style="font-size:0.85rem;color:var(--text-dim);">—</div>
{% endif %}
</div>
{% endfor %}
</div>
{% else %}
<p style="font-size:0.8rem;color:var(--text-dim);margin:0;">No readings yet — waiting for the next poll.</p>
{% endif %}
</div>
{% endfor %}
plugins/snmp/templates/snmp/index.html
+45 -43
View File
@@ -1,55 +1,57 @@
{# plugins/snmp/templates/snmp/index.html — every SNMP device, those not
answering first, each saying what its newest poll found: its readings in
their units, or why it didn't answer (milestone 491). #}
{# plugins/snmp/templates/snmp/index.html #}
{% extends "base.html" %}
{% from "_macros.html" import page_header, status_badge, stat_tile, empty_state %}
{% block title %}SNMP — Steward{% endblock %}
{% block content %}
{% set admin = session.user_role == 'admin' %}
{% call page_header("SNMP devices", (devices|length ~ " device" ~ ("s" if devices|length != 1 else "")) if devices else none) %}
{% if admin %}<a class="btn btn-sm" href="/plugins/snmp/devices/new">Add device</a>{% endif %}
{% endcall %}
<div style="display:flex;align-items:center;gap:1rem;margin-bottom:1.5rem;">
<h1 class="page-title" style="margin:0;">SNMP Devices</h1>
<span style="font-size:0.8rem;color:var(--text-dim);">{{ devices|length }} device(s) configured</span>
</div>
{% if not devices %}
<div class="card">{{ empty_state("No SNMP devices yet. Add a switch, a UPS, a printer or a server that answers SNMP, and Steward polls it every minute.",
"Add device" if admin else none, "/plugins/snmp/devices/new") }}</div>
{% endif %}
<div class="card" style="color:var(--text-muted);text-align:center;padding:2rem;">
No SNMP devices configured. Add devices to <code>plugin.yaml</code>.
</div>
{% else %}
{% for d in devices %}
<section class="card" aria-labelledby="snmp-{{ d.id }}">
<div class="host-health-head">
{% if d.reach == 'up' %}{{ status_badge('up', 'Answering') }}
{% elif d.reach == 'down' %}{{ status_badge('down', 'Not answering') }}
{% elif d.reach == 'stale' %}{{ status_badge('stale', 'Not polled') }}
{% else %}{{ status_badge('pending', 'Not polled yet') }}{% endif %}
<h2 id="snmp-{{ d.id }}" class="section-title" style="margin:0;"><a href="/plugins/snmp/devices/{{ d.id }}">{{ d.name }}</a></h2>
<span class="field-hint mono">{{ d.address }}{% if d.port != 161 %}:{{ d.port }}{% endif %}</span>
<span class="field-hint">SNMP v{{ d.version }}</span>
<span class="page-actions" style="margin-left:auto;">
<a href="/plugins/snmp/devices/{{ d.id }}" class="btn btn-sm btn-ghost">History</a>
{% if admin %}<a href="/plugins/snmp/devices/{{ d.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>{% endif %}
</span>
{% for device in devices %}
<div class="card" style="margin-bottom:1rem;">
<div style="display:flex;align-items:center;gap:1rem;margin-bottom:1rem;">
<h2 style="margin:0;font-size:1rem;font-weight:600;">{{ device.name }}</h2>
<span style="font-size:0.78rem;color:var(--text-dim);">{{ device.host }}</span>
{% if device.readings %}
<span style="font-size:0.75rem;color:var(--green);margin-left:auto;">&#9679; reachable</span>
{% else %}
<span style="font-size:0.75rem;color:var(--text-dim);margin-left:auto;">&#9675; no data yet</span>
{% endif %}
<a href="/plugins/snmp/device/{{ device.name }}" class="btn btn-ghost" style="font-size:0.78rem;padding:0.2rem 0.6rem;">History</a>
</div>
{% if d.reach == 'down' %}
<p class="tone-crit" style="margin:0 0 0.4rem;">{{ d.says }}</p>
<p class="field-hint" style="margin:0 0 0.6rem;">
{%- if d.ok_at %}Last answered {{ d.ok_at|ago }}; the readings below are from then.{% else %}It has never answered.{% endif %}
Tried {{ d.polled_at|ago }}.</p>
{% elif d.reach == 'stale' %}
<p class="field-hint" style="margin:0 0 0.6rem;">{{ d.says }} Last polled {{ d.polled_at|ago }}.</p>
{% elif d.reach == 'pending' %}
<p class="field-hint" style="margin:0 0 0.6rem;">Not polled yet: the first poll runs within a minute of saving.</p>
{% endif %}
{% if d.reach != 'pending' and d.readings %}
<div class="stat-strip" style="margin-bottom:0;">
{% for r in d.readings %}{{ stat_tile(r.name, r.text, hint="from the second poll" if r.counter and r.text == "—" else none) }}{% endfor %}
{% if device.readings %}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(200px,1fr));gap:0.75rem;">
{% for oid_cfg in device.oids %}
{% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
{% set val = device.readings.get(label) %}
<div style="background:var(--bg-alt);border-radius:6px;padding:0.6rem 0.9rem;">
<div style="font-size:0.72rem;color:var(--text-dim);margin-bottom:0.2rem;font-family:monospace;">{{ label }}</div>
{% if val is not none %}
<div style="font-size:1.1rem;font-weight:600;font-variant-numeric:tabular-nums;">
{% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}<span style="font-size:0.75rem;color:var(--text-muted);">M</span>
{% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}<span style="font-size:0.75rem;color:var(--text-muted);">K</span>
{% else %}{{ "%.4g"|format(val) }}{% endif %}
</div>
{% else %}
<div style="font-size:0.85rem;color:var(--text-dim);">—</div>
{% endif %}
</div>
{% endfor %}
</div>
{% else %}
<p style="font-size:0.85rem;color:var(--text-muted);margin:0;">
No readings yet — waiting for next poll cycle.
</p>
{% endif %}
{% if d.missing %}
<p class="field-hint" style="margin:0.6rem 0 0;">Not answered by this device: {{ d.missing|join(", ") }}. It may not support them; Edit to remove them.</p>
{% endif %}
</section>
</div>
{% endfor %}
{% endif %}
{% endblock %}
plugins/snmp/templates/snmp/widget.html
+47 -36
View File
@@ -1,38 +1,49 @@
{# plugins/snmp/templates/snmp/widget.html — the dashboard SNMP widget (#5255,
milestone 491): a verdict, devices not answering first with why, then the
rest, each with two readings in their units. The name links to its page. #}
{% from "_macros.html" import status_dot %}
{% if not total %}
<div class="w-verdict" style="color:var(--text-muted);">No SNMP devices yet.{% if session.user_role == 'admin' %} <a href="/plugins/snmp/devices/new">Add one</a>{% endif %}</div>
{% else %}
{% set down = devices | selectattr("reach", "equalto", "down") | list %}
<div class="w-verdict {{ 'tone-crit' if counts.down else ('tone-warn' if counts.stale or counts.pending else 'tone-ok') }}">
{%- if counts.down == 1 and down %}{{ down[0].name }} isn't answering.
{%- elif counts.down %}{{ counts.down }} of {{ total }} devices not answering.
{%- elif counts.stale %}{{ counts.stale }} not polled recently.
{%- elif counts.pending %}{{ counts.pending }} not polled yet, {{ counts.up }} answering.
{%- else %}All {{ total }} device{{ 's' if total != 1 }} answering.{% endif -%}
{# plugins/snmp/templates/snmp/widget.html #}
{% if devices %}
{% for device in devices %}
<div class="ping-row" style="align-items:flex-start;flex-direction:column;gap:0.3rem;padding:0.4rem 0;">
<div style="display:flex;align-items:center;gap:0.6rem;width:100%;">
<span style="font-size:0.82rem;font-weight:500;flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">
{{ device.name }}
</span>
{% if device.readings %}
<span style="font-size:0.7rem;color:var(--green);">&#9679;</span>
{% else %}
<span style="font-size:0.7rem;color:var(--text-dim);">&#9675;</span>
{% endif %}
<a href="/plugins/snmp/device/{{ device.name }}" style="font-size:0.72rem;color:var(--text-muted);">detail →</a>
</div>
{% if device.readings %}
<div style="display:flex;flex-wrap:wrap;gap:0.5rem 1rem;padding-left:0.1rem;">
{% for oid_cfg in device.oids[:4] %}
{% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
{% set val = device.readings.get(label) %}
{% if val is not none %}
<span style="font-size:0.78rem;font-variant-numeric:tabular-nums;">
<span style="color:var(--text-muted);">{{ label }}</span>
<span style="margin-left:0.25rem;">
{% if val >= 1000000 %}{{ "%.1f"|format(val / 1000000) }}M
{% elif val >= 1000 %}{{ "%.0f"|format(val / 1000) }}K
{% else %}{{ "%.4g"|format(val) }}{% endif %}
</span>
</span>
{% endif %}
{% endfor %}
{% if device.oids|length > 4 %}
<span style="font-size:0.72rem;color:var(--text-dim);">+{{ device.oids|length - 4 }} more</span>
{% endif %}
</div>
{% endif %}
</div>
<table class="w-table">
<colgroup><col style="width:40%"><col><col class="w-wide-only" style="width:6rem"></colgroup>
<thead><tr><th>Device</th><th>Readings</th><th class="w-wide-only">Polled</th></tr></thead>
<tbody>
{% for d in devices %}
<tr>
<td class="w-name" title="{{ d.name }}">
{{- status_dot({'down': 'down', 'stale': 'stale', 'pending': 'pending'}.get(d.reach, 'up')) -}}
{% if session.user_id %}<a href="/plugins/snmp/devices/{{ d.id }}">{{ d.name }}</a>{% else %}{{ d.name }}{% endif %}
{%- if d.reach == 'down' %}<span class="w-sub tone-crit" title="{{ d.says }}">{% if d.ok_at %}no reply since {{ d.ok_at|ago }}{% else %}never answered{% endif %}</span>
{%- elif d.reach == 'stale' %}<span class="w-sub">not polled since {{ d.polled_at|ago }}</span>
{%- elif d.reach == 'pending' %}<span class="w-sub">not polled yet</span>{% endif %}
</td>
<td title="{% for r in d.key %}{{ r.name }} {{ r.text }}{% if not loop.last %} · {% endif %}{% endfor %}">
{%- for r in d.key %}<span style="color:var(--text-muted);">{{ r.name }}</span> {{ r.text }}{% if not loop.last %} · {% endif %}{% else %}—{% endfor -%}
</td>
<td class="w-wide-only">{% if d.polled_at %}{{ d.polled_at|ago }}{% else %}—{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% if more %}<div class="w-more">+{{ more }} more{% if session.user_id %} · <a href="/plugins/snmp/">all devices</a>{% endif %}</div>{% endif %}
{% if not loop.last %}
<div style="height:1px;background:var(--border-low);margin:0.1rem 0;"></div>
{% endif %}
{% endfor %}
{% if total > devices|length %}
<div style="color:var(--text-dim);font-size:0.75rem;padding:0.25rem 0;">
+{{ total - devices|length }} more — <a href="/plugins/snmp/">view all →</a>
</div>
{% endif %}
{% else %}
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No SNMP devices configured.</p>
{% endif %}
plugins/traefik/__init__.py
-13
View File
@@ -12,11 +12,6 @@ def setup(app: "Quart") -> None:
global _app
_app = app
from .models import TraefikMetric, TraefikCert, TraefikGlobalStat, TraefikAccessSummary # noqa: F401
from steward.core.cleanup import prune_by_age
# Samples older than data.retention_days are pruned by the cleanup (#5155).
prune_by_age(TraefikMetric.scraped_at)
prune_by_age(TraefikGlobalStat.scraped_at)
prune_by_age(TraefikAccessSummary.period_end)
def get_scheduled_tasks() -> list:
@@ -38,11 +33,3 @@ def get_blueprint():
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (Traefik services view).
return [{"label": "Traefik", "href": "/plugins/traefik/"}]
def get_alert_metrics() -> dict[str, list[str]]:
"""What scheduler.py records for alert rules: per router (resource = the
router name) and cert_expiry_days per certificate (resource = its CN)."""
return {"traefik": ["request_rate", "error_rate_4xx_pct", "error_rate_5xx_pct",
"latency_p50_ms", "latency_p95_ms", "latency_p99_ms",
"response_bytes_rate", "cert_expiry_days"]}
plugins/traefik/migrations/versions/traefik_004_covering_indexes.py
-34
View File
@@ -1,34 +0,0 @@
"""Traefik reads answered from the index alone
The rows page and the request chart read every raw row in the chosen range.
With only (router_name, scraped_at) indexed, each of those rows was a heap
fetch: a random disk read on a database whose indexes no longer fit in
memory, which took the rows page to 12-28 s. Carrying the charted columns
in the index makes these index-only scans; scraped_at gets an index of its
own for the request chart, which filters on time across every router.
Revision ID: traefik_004_covering_indexes
Revises: traefik_003_access_log
Create Date: 2026-10-08
"""
from typing import Sequence, Union
from alembic import op
revision: str = "traefik_004_covering_indexes"
down_revision: Union[str, None] = "traefik_003_access_log"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics")
op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics", ["router_name", "scraped_at"],
postgresql_include=["request_rate", "latency_p95_ms", "error_rate_5xx_pct"])
op.create_index("ix_traefik_metrics_scraped", "traefik_metrics", ["scraped_at"],
postgresql_include=["request_rate", "error_rate_5xx_pct"])
def downgrade() -> None:
op.drop_index("ix_traefik_metrics_scraped", "traefik_metrics")
op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics")
op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics", ["router_name", "scraped_at"])
plugins/traefik/migrations/versions/traefik_005_reload_at.py
-41
View File
@@ -1,41 +0,0 @@
"""Traefik's last good config reload is a moment; routers are graded on a window
- traefik_config_last_reload_success is the Unix time of the last successful
reload. It was stored as a float and read as 1.0 = success, so the page said
"Reload failed" on every scrape (#5355). The column becomes the moment; the
old values were timestamps misread, so they go with it.
- Routers are now graded on their last 15 minutes (routes.recent_window),
which reads the 4xx share too: the covering router index carries it, so
that read stays index-only.
Revision ID: traefik_005_reload_at
Revises: traefik_004_covering_indexes
Create Date: 2026-10-08
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "traefik_005_reload_at"
down_revision: Union[str, None] = "traefik_004_covering_indexes"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
_CHARTED = ["request_rate", "latency_p95_ms", "error_rate_5xx_pct"]
def upgrade() -> None:
op.drop_column("traefik_global_stats", "config_last_reload_success")
op.add_column("traefik_global_stats",
sa.Column("config_last_reload_at", sa.DateTime(timezone=True), nullable=True))
op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics")
op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics", ["router_name", "scraped_at"],
postgresql_include=_CHARTED + ["error_rate_4xx_pct"])
def downgrade() -> None:
op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics")
op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics", ["router_name", "scraped_at"],
postgresql_include=_CHARTED)
op.drop_column("traefik_global_stats", "config_last_reload_at")
op.add_column("traefik_global_stats", sa.Column("config_last_reload_success", sa.Float, nullable=True))
plugins/traefik/models.py
+5 -7
View File
@@ -1,16 +1,16 @@
# plugins/traefik/models.py
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from sqlalchemy import DateTime, Float, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from steward.models.base import Base
from steward.core.ids import time_ordered_id
class TraefikMetric(Base):
__tablename__ = "traefik_metrics"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
router_name: Mapped[str] = mapped_column(String(255), nullable=False)
scraped_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
@@ -41,7 +41,7 @@ class TraefikAccessSummary(Base):
"""Aggregated traffic-origin summary — one row per scheduler tick."""
__tablename__ = "traefik_access_summaries"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
period_start: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
period_end: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
total_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
@@ -57,13 +57,11 @@ class TraefikGlobalStat(Base):
"""One row per scrape — process-level and config stats."""
__tablename__ = "traefik_global_stats"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
scraped_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
)
open_conns_total: Mapped[float | None] = mapped_column(Float, nullable=True)
config_reloads_total: Mapped[float | None] = mapped_column(Float, nullable=True)
# When Traefik last loaded its configuration successfully: its
# traefik_config_last_reload_success gauge is that moment, not a 0/1 flag.
config_last_reload_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
config_last_reload_success: Mapped[float | None] = mapped_column(Float, nullable=True)
process_memory_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)
plugins/traefik/plugin.yaml
-3
View File
@@ -14,9 +14,6 @@ tags:
config:
metrics_url: "http://localhost:8080/metrics"
# Traefik's API, for which hosts each router serves (so a certificate's
# expiry shows on its router). Empty: /api on the metrics URL's host.
api_url: ""
scrape_interval_seconds: 60
access_log:
enabled: false
plugins/traefik/routes.py
+125 -386
View File
@@ -1,21 +1,15 @@
# plugins/traefik/routes.py
from __future__ import annotations
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from datetime import datetime, timezone
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import func, select
from sqlalchemy import Integer, cast, func, select
import json
import time
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample, time_bucket
from steward.core.formatting import cert_left, status_class
from steward.models.metrics import PluginMetricLatest
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric
from .scheduler import router_domains
from .scraper import cert_names, covers
traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
@@ -42,189 +36,14 @@ def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
)
_GRADE_ORDER = {"crit": 0, "warn": 1, "ok": 2}
def error_cutoffs(thresholds: dict) -> tuple[float, float]:
"""(5xx, 4xx): the error shares, in percent, past which a router is failing
or worth a look. Settings → Thresholds, Traefik routers (#5380)."""
return (float(thresholds.get("error_5xx", {}).get("crit", 0.1)),
float(thresholds.get("error_4xx", {}).get("warn", 1.0)))
def router_state(reading, thresholds: dict, over: str = "") -> tuple[str, str | None]:
"""(grade, why) for a router's reading (recent_window's, or one scrape's):
crit while it serves 5xx past its cutoff, warn on a p95 past the latency
critical cutoff or a 4xx share past its cutoff (all Settings → Thresholds),
otherwise ok with nothing to say. `over` names the span the reading
covers, for the why."""
error_5xx, error_4xx = error_cutoffs(thresholds)
if (reading.error_rate_5xx_pct or 0) > error_5xx:
return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}"
p95 = reading.latency_p95_ms
if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
return "warn", f"p95 latency is {p95:.0f} ms{over}"
if (reading.error_rate_4xx_pct or 0) > error_4xx:
return "warn", f"{reading.error_rate_4xx_pct:.1f}% of responses are 4xx{over}"
return "ok", None
# A router is graded on its last STATE_WINDOW of traffic, not on one scrape:
# a share near a cutoff crossed it on one scrape and fell back on the next,
# so problem cards came and went every poll (#5214). Over the window a real
# problem stays listed until it has been clean a while, and a one-scrape
# blip is diluted rather than shown.
STATE_WINDOW = timedelta(minutes=15)
STATE_WINDOW_WORDS = " over the last 15 min"
async def recent_window(db, names: list[str], now: datetime) -> dict[str, SimpleNamespace]:
"""{router: one reading of its last STATE_WINDOW}: the 4xx and 5xx shares
weighted by each scrape's traffic (an idle scrape's 100% counts for
little), p95 averaged. A router with no rows in the window is absent."""
if not names:
return {}
rate = TraefikMetric.request_rate
result = await db.execute(
select(
TraefikMetric.router_name,
func.sum(rate).label("req"),
func.sum(rate * TraefikMetric.error_rate_4xx_pct).label("e4w"),
func.sum(rate * TraefikMetric.error_rate_5xx_pct).label("e5w"),
func.avg(TraefikMetric.error_rate_4xx_pct).label("e4"),
func.avg(TraefikMetric.error_rate_5xx_pct).label("e5"),
func.avg(TraefikMetric.latency_p95_ms).label("p95"),
)
.where(TraefikMetric.router_name.in_(names))
.where(TraefikMetric.scraped_at >= now - STATE_WINDOW)
.group_by(TraefikMetric.router_name)
)
out = {}
for r in result.all():
req = r.req or 0.0
out[r.router_name] = SimpleNamespace(
error_rate_4xx_pct=r.e4w / req if req > 0 else r.e4,
error_rate_5xx_pct=r.e5w / req if req > 0 else r.e5,
latency_p95_ms=r.p95,
)
return out
def grade_routers(routers: list[dict], window: dict, thresholds: dict) -> None:
"""Set each router's state and why from its recent window, or from its
latest scrape when the window holds nothing."""
for r in routers:
reading = window.get(r["name"])
r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS) if reading
else router_state(r["latest"], thresholds))
# A router missing from this many scrapes in a row is no longer reported:
# Traefik dropped it (a removed service), so its last state is history.
GONE_AFTER_SCRAPES = 3
def gone_routers(routers: list[dict], scrape_seconds: int) -> tuple[list[dict], list[dict]]:
"""(current, gone): a router whose newest row trails the newest scrape of
any router by GONE_AFTER_SCRAPES intervals is gone, newest-seen first."""
if not routers:
return [], []
newest = max(r["latest"].scraped_at for r in routers)
cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
current = [r for r in routers if r["latest"].scraped_at >= cutoff]
gone = sorted((r for r in routers if r["latest"].scraped_at < cutoff),
key=lambda r: r["latest"].scraped_at, reverse=True)
return current, gone
def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
"""(problems, healthy): failing routers before slow or noisy ones, each
group by name; healthy routers by name."""
problems = sorted((r for r in routers if r["state"] != "ok"),
key=lambda r: (_GRADE_ORDER[r["state"]], r["name"].lower()))
healthy = sorted((r for r in routers if r["state"] == "ok"), key=lambda r: r["name"].lower())
return problems, healthy
# The figures the scraper records for every router (scheduler._do_scrape).
ROUTER_METRICS = ("request_rate", "error_rate_4xx_pct", "error_rate_5xx_pct", "latency_p50_ms",
"latency_p95_ms", "latency_p99_ms", "response_bytes_rate")
async def latest_per_router(db) -> list[SimpleNamespace]:
"""Each router's newest scrape, read from plugin_metric_latest: the scraper
records every figure through MetricBatch, which keeps one row per router
and figure there (latest_by_resource's table, #5208). A walk of
traefik_metrics' index, one step per router, took 7 s on the live
instance (#5355); each step lands on the router's oldest entries, which
retention deletes every minute, so it likely waded through dead ones.
Each router reads as one scrape, with the attributes a TraefikMetric row
has; certificates, recorded under the same source, are left out."""
rows = (await db.execute(select(PluginMetricLatest).where(
PluginMetricLatest.source_module == "traefik",
PluginMetricLatest.metric_name.in_(ROUTER_METRICS)))).scalars()
by_router: dict[str, dict] = {}
for r in rows:
by_router.setdefault(r.resource_name, {})[r.metric_name] = r
return [SimpleNamespace(router_name=name, scraped_at=max(m.recorded_at for m in figures.values()),
**{k: figures[k].value if k in figures else 0.0 for k in ROUTER_METRICS})
for name, figures in by_router.items() if "request_rate" in figures]
def current_certs(certs, scrape_seconds: int) -> tuple[list, list]:
"""(served, old): a certificate is served while Traefik still reports it.
Rows are upserted by serial and never deleted, so a renewed or retired
certificate lingers; one missing from the last GONE_AFTER_SCRAPES
scrapes is old (#5254)."""
certs = list(certs)
if not certs:
return [], []
newest = max(c.scraped_at for c in certs)
cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
return ([c for c in certs if c.scraped_at >= cutoff],
sorted((c for c in certs if c.scraped_at < cutoff), key=lambda c: c.scraped_at, reverse=True))
def router_hosts(name: str, domains: dict[str, list[str]]) -> list[str]:
"""The hosts a metrics router name serves. The API names a router
"app@docker"; metrics may carry it as "websecure-app@docker", one per
entrypoint, so a name not found as-is is tried without that prefix."""
if name in domains:
return domains[name]
for api_name, hosts in domains.items():
if name.endswith("-" + api_name):
return hosts
return []
def certs_by_router(routers: list[str], domains: dict[str, list[str]], certs, now) -> dict[str, dict]:
"""{router: {"cn", "days"}}: the soonest-expiring served certificate valid
for one of the router's hosts. A router with no hosts, or none covered,
is absent."""
out: dict[str, dict] = {}
names = [(c, cert_names(c.cn, c.sans)) for c in certs]
for router in routers:
for host in router_hosts(router, domains):
for c, cn in names:
if covers(cn, host):
days = (c.not_after - now).total_seconds() / 86400.0
if router not in out or days < out[router]["days"]:
out[router] = {"cn": c.cn or host, "days": days}
return out
def _access_log() -> dict:
"""The plugin's access_log settings ({} when unset or malformed)."""
cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
return cfg if isinstance(cfg, dict) else {}
@traefik_bp.get("/")
@require_role(UserRole.viewer)
async def index():
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
access_log_enabled = bool(_access_log().get("enabled", False))
_al_cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
if not isinstance(_al_cfg, dict):
_al_cfg = {}
access_log_enabled = _al_cfg.get("enabled", False)
current_range = request.args.get("range", DEFAULT_RANGE)
return await render_template(
"traefik/index.html",
@@ -240,31 +59,57 @@ async def rows():
"""HTMX fragment: service cards with sparklines scoped to selected time range."""
since, range_key = parse_range(request.args.get("range"))
now = datetime.now(timezone.utc)
thresholds = current_app.config.get("THRESHOLDS", {})
scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
phases = _Phases()
async with current_app.db_sessionmaker() as db:
# Every known router, with its most recent raw row; a router Traefik
# stopped reporting gets no sparklines, so its history is not read.
newest = await latest_per_router(db)
current, gone = gone_routers([{"name": m.router_name, "latest": m} for m in newest], scrape)
phases.mark("latest")
# All known routers
result = await db.execute(
select(TraefikMetric.router_name)
.distinct()
.order_by(TraefikMetric.router_name)
)
routers = [row[0] for row in result.all()]
names = [r["name"] for r in current]
history = await router_history(db, names, since, bucket_seconds(since))
phases.mark("history")
grade_routers(current, await recent_window(db, names, now), thresholds)
phases.mark("window")
for r in current:
points = history[r["name"]]
r["sparkline_req"] = _sparkline([p.request_rate for p in points])
r["sparkline_bytes"] = _sparkline([p.response_bytes_rate for p in points])
r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points])
r["sparkline_4xx"] = _sparkline([p.error_rate_4xx_pct for p in points])
r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points])
phases.mark("sparklines")
b_secs = bucket_seconds(since)
bucket_col = (
cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs
).label("bucket")
router_data = []
for router in routers:
# Bucketed history for sparklines — always ~80 points regardless of range
result = await db.execute(
select(
func.avg(TraefikMetric.request_rate).label("request_rate"),
func.avg(TraefikMetric.latency_p95_ms).label("latency_p95_ms"),
func.avg(TraefikMetric.error_rate_5xx_pct).label("error_rate_5xx_pct"),
func.min(TraefikMetric.scraped_at).label("scraped_at"),
bucket_col,
)
.where(TraefikMetric.router_name == router)
.where(TraefikMetric.scraped_at >= since)
.group_by(bucket_col)
.order_by(bucket_col)
)
history = result.all()
# Latest value — always the most recent raw row
result = await db.execute(
select(TraefikMetric)
.where(TraefikMetric.router_name == router)
.order_by(TraefikMetric.scraped_at.desc())
.limit(1)
)
latest = result.scalar_one_or_none()
if not latest:
continue
router_data.append({
"name": router,
"latest": latest,
"sparkline_req": _sparkline([r.request_rate for r in history]),
"sparkline_p95": _sparkline([r.latency_p95_ms for r in history]),
"sparkline_5xx": _sparkline([r.error_rate_5xx_pct for r in history]),
})
# Latest global stat (always most recent, not range-filtered)
result = await db.execute(
@@ -279,104 +124,77 @@ async def rows():
select(TraefikCert).order_by(TraefikCert.not_after)
)
raw_certs = result.scalars().all()
phases.mark("stats_certs")
served, old = current_certs(raw_certs, scrape)
domains = router_domains()
def cert_row(c):
names = cert_names(c.cn, c.sans)
return {
certs = []
for c in raw_certs:
days = (c.not_after - now).total_seconds() / 86400.0
certs.append({
"serial": c.serial,
"cn": c.cn or c.serial,
"sans": [n for n in names if n != c.cn],
"sans": c.sans,
"not_after": c.not_after,
"days_remaining": (c.not_after - now).total_seconds() / 86400.0,
# The routers whose hosts it covers; None when Traefik's API
# couldn't say which hosts each router serves.
"routers": None if domains is None else sorted(
{r["name"].split("@")[0] for r in current
for h in router_hosts(r["name"], domains) if covers(names, h)}),
}
"days_remaining": days,
})
certs = [cert_row(c) for c in served]
old_certs = [cert_row(c) for c in old]
problems, healthy = split_routers(current)
phases.mark("cert_rows")
error_5xx, error_4xx = error_cutoffs(thresholds)
page = await render_template(
return await render_template(
"traefik/rows.html",
problems=problems,
healthy=healthy,
gone=gone,
error_4xx=error_4xx,
error_5xx=error_5xx,
router_data=router_data,
global_stat=global_stat,
certs=certs,
old_certs=old_certs,
range_key=range_key,
)
phases.mark("render")
phases.report("rows", f"routers={len(current)} range={range_key}")
return page
@traefik_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""The dashboard Routers widget (#5254), in the widget-shape order: a
verdict, routers in trouble with why, then the busiest healthy ones under
one header. Only routers and certificates Traefik still reports. A
certificate's expiry shows on the router it serves, and makes that router
a problem once it is inside the Settings → Thresholds TLS window."""
"""HTMX dashboard widget fragment."""
now = datetime.now(timezone.utc)
thresholds = current_app.config.get("THRESHOLDS", {})
scrape = current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
phases = _Phases()
async with current_app.db_sessionmaker() as db:
latest = await latest_per_router(db)
phases.mark("latest")
current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
window = await recent_window(db, [r["name"] for r in current], now)
phases.mark("window")
certs = (await db.execute(select(TraefikCert))).scalars().all()
phases.mark("certs")
phases.report("widget", f"routers={len(current)}")
result = await db.execute(
select(TraefikMetric.router_name).distinct().order_by(TraefikMetric.router_name)
)
routers = [row[0] for row in result.all()]
router_data = []
for router in routers:
result = await db.execute(
select(TraefikMetric)
.where(TraefikMetric.router_name == router)
.order_by(TraefikMetric.scraped_at.desc())
.limit(1)
)
latest = result.scalar_one_or_none()
if latest:
router_data.append({"name": router, "latest": latest})
router_data.sort(key=lambda r: (
-(1 if r["latest"].error_rate_5xx_pct > 0.1 else 0),
-(1 if r["latest"].error_rate_4xx_pct > 5.0 else 0),
-r["latest"].request_rate,
))
total_routers = len(router_data)
router_data = router_data[:5]
result = await db.execute(
select(TraefikCert).order_by(TraefikCert.not_after)
)
raw_certs = result.scalars().all()
expiring_certs = []
for c in raw_certs:
days = (c.not_after - now).total_seconds() / 86400.0
if days < 30:
expiring_certs.append({"cn": c.cn or c.serial, "days_remaining": days})
served, _old = current_certs(certs, scrape)
domains = router_domains()
by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
grade_routers(current, window, thresholds)
for r in current:
r["cert"] = by_router.get(r["name"])
if r["cert"]:
grade = status_class(r["cert"]["days"], "tls_days", thresholds)
r["cert"]["grade"] = grade
if _GRADE_ORDER.get(grade, 2) < _GRADE_ORDER[r["state"]]:
left = cert_left(r["cert"]["days"]) # "12 days", "expires today", "expired …"
r["state"], r["why"] = grade, (
f"certificate expires in {left}" if left[0].isdigit() else f"certificate {left}")
problems, healthy = split_routers(current)
healthy.sort(key=lambda r: -(r["latest"].request_rate or 0))
# With no router domains to go on, certificates can't sit on their
# routers: the expiring ones are listed after the routers instead.
loose = [] if domains is not None else sorted(
({"cn": c.cn or c.serial, "days": (c.not_after - now).total_seconds() / 86400.0} for c in served),
key=lambda c: c["days"])
loose = [c for c in loose if status_class(c["days"], "tls_days", thresholds) in ("warn", "crit")]
return await render_template(
"traefik/widget.html",
problems=problems, healthy=healthy[:ROUTERS_SHOWN],
healthy_more=max(0, len(healthy) - ROUTERS_SHOWN),
total=len(current), loose_certs=loose, domains_known=domains is not None,
router_data=router_data,
expiring_certs=expiring_certs,
total_routers=total_routers,
)
# Healthy routers on the dashboard widget (the busiest); the page lists all.
ROUTERS_SHOWN = 6
@traefik_bp.get("/widget/access_log")
@require_role(UserRole.viewer)
async def widget_access_log():
@@ -430,112 +248,38 @@ async def widget_access_log():
summary=summary, widget_id=widget_id)
# A rows or widget request past this logs where its time went, phase by
# phase, so a slow query names itself (#5355: 8 s on every poll, warm).
SLOW_PAGE_SECONDS = 1.0
class _Phases:
"""Wall-clock per phase of one request; report() logs them when slow."""
def __init__(self):
self.start = self.last = time.monotonic()
self.took: dict[str, float] = {}
def mark(self, name: str) -> None:
now = time.monotonic()
self.took[name] = now - self.last
self.last = now
def report(self, what: str, detail: str = "") -> None:
total = time.monotonic() - self.start
if total >= SLOW_PAGE_SECONDS:
current_app.logger.warning("traefik %s: slow %.1fs (%s) %s", what, total,
" ".join(f"{k}={v:.2f}s" for k, v in self.took.items()), detail)
async def router_history(db, names: list[str], since, bucket_secs: int) -> dict[str, list]:
"""{router: [rows of request_rate, latency_p95_ms, error_rate_5xx_pct,
error_rate_4xx_pct, response_bytes_rate]}
per `bucket_secs` window since `since`, oldest first: one query for every
router, answered from the covering (router_name, scraped_at) index
(traefik_004). It was a query per router, each row a heap read, which
took the rows page to 12-28 s."""
history: dict[str, list] = {n: [] for n in names}
if not history:
return history
bucket_col = time_bucket(TraefikMetric.scraped_at, bucket_secs)
result = await db.execute(
select(
TraefikMetric.router_name,
func.avg(TraefikMetric.request_rate).label("request_rate"),
func.avg(TraefikMetric.latency_p95_ms).label("latency_p95_ms"),
func.avg(TraefikMetric.error_rate_5xx_pct).label("error_rate_5xx_pct"),
func.avg(TraefikMetric.error_rate_4xx_pct).label("error_rate_4xx_pct"),
func.avg(TraefikMetric.response_bytes_rate).label("response_bytes_rate"),
bucket_col,
)
.where(TraefikMetric.router_name.in_(names))
.where(TraefikMetric.scraped_at >= since)
.group_by(TraefikMetric.router_name, bucket_col)
.order_by(TraefikMetric.router_name, bucket_col)
)
for row in result.all():
history[row.router_name].append(row)
return history
async def request_history(db, since, bucket_seconds: int) -> list[dict]:
"""[{"at", "req_rate", "err_pct"}] per time bucket since `since`: each
scrape summed over routers (the total req/s), averaged over the bucket;
the 5xx share weighted by each router's traffic, so a quiet router's
errors don't count as much as a busy one's."""
per_scrape = (
select(
TraefikMetric.scraped_at.label("at"),
func.sum(TraefikMetric.request_rate).label("req"),
func.sum(TraefikMetric.request_rate * TraefikMetric.error_rate_5xx_pct).label("err_weighted"),
)
.where(TraefikMetric.scraped_at >= since)
.group_by(TraefikMetric.scraped_at)
.subquery()
)
bucket_col = time_bucket(per_scrape.c.at, bucket_seconds)
rows = (await db.execute(
select(
func.avg(per_scrape.c.req).label("req_rate"),
func.sum(per_scrape.c.err_weighted).label("err_weighted"),
func.sum(per_scrape.c.req).label("req_sum"),
func.min(per_scrape.c.at).label("at"),
bucket_col,
)
.group_by(bucket_col)
.order_by(bucket_col)
)).all()
return [{"at": r.at, "req_rate": round(r.req_rate or 0, 3),
"err_pct": round((r.err_weighted or 0) / r.req_sum, 2) if r.req_sum else 0.0}
for r in rows]
@traefik_bp.get("/widget/request_chart")
@require_role(UserRole.viewer)
async def widget_request_chart():
"""HTMX dashboard widget: requests per second across all routers, and the
share of them answered 5xx, over the last few hours (#5256). Each scrape
is summed over routers first (the total), then averaged per bucket; the
5xx share is weighted by each router's traffic, not averaged per router."""
"""HTMX dashboard widget: Chart.js request rate + error % over time."""
from datetime import timedelta
hours = max(1, min(24, int(request.args.get("hours", 6) or 6)))
widget_id = request.args.get("wid", "0")
since = datetime.now(timezone.utc) - timedelta(hours=hours)
b_secs = max(60, (hours * 3600) // 80) # ~80 buckets
async with current_app.db_sessionmaker() as db:
history = await request_history(db, since, b_secs)
bucket_col = (
cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs
).label("bucket")
labels = [int(r["at"].timestamp() * 1000) for r in history] # epoch ms; the browser shows local time
req_rates = [r["req_rate"] for r in history]
err_pcts = [r["err_pct"] for r in history]
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(
func.avg(TraefikMetric.request_rate).label("req_rate"),
func.avg(TraefikMetric.error_rate_5xx_pct).label("err_pct"),
func.min(TraefikMetric.scraped_at).label("scraped_at"),
bucket_col,
)
.where(TraefikMetric.scraped_at >= since)
.group_by(bucket_col)
.order_by(bucket_col)
)
history = result.all()
labels = list(range(len(history)))
req_rates = [round(r.req_rate or 0, 3) for r in history]
err_pcts = [round(r.err_pct or 0, 2) for r in history]
return await render_template(
"traefik/widget_request_chart.html",
@@ -544,9 +288,6 @@ async def widget_request_chart():
error_rate_json=json.dumps(err_pcts),
widget_id=widget_id,
hours=hours,
now_req=req_rates[-1] if req_rates else None,
peak_req=max(req_rates) if req_rates else None,
peak_err=max(err_pcts) if err_pcts else None,
)
@@ -567,10 +308,8 @@ async def traffic():
rows = result.scalars().all()
if not rows:
al = _access_log()
return await render_template(
"traefik/traffic.html", summary=None, history=[], range_key=range_key,
access_log_enabled=bool(al.get("enabled", False)), access_log_path=al.get("path", ""),
"traefik/traffic.html", summary=None, history=[], range_key=range_key
)
total = sum(r.total_requests for r in rows)
plugins/traefik/scheduler.py
+6 -29
View File
@@ -16,16 +16,6 @@ logger = logging.getLogger(__name__)
_prev_metrics: dict = {}
_prev_time: float = 0.0
# Router name -> host names from Traefik's API, refreshed each scrape (#5254).
# None until the API has answered once, and again whenever it stops answering,
# so readers can tell "no domains" from "don't know".
_router_domains: dict[str, list[str]] | None = None
def router_domains() -> dict[str, list[str]] | None:
return _router_domains
# Access log state
_geoip_db = None
_access_log_last_tick: "datetime | None" = None
@@ -48,19 +38,17 @@ def make_scrape_task(app: "Quart") -> ScheduledTask:
async def _do_scrape(app: "Quart") -> None:
"""Fetch Traefik metrics, write to DB, and emit plugin_metrics."""
global _prev_metrics, _prev_time, _router_domains
global _prev_metrics, _prev_time
from datetime import datetime, timezone
from .models import TraefikCert, TraefikGlobalStat, TraefikMetric
from .scraper import (
api_base,
compute_global_stats,
compute_router_metrics,
extract_certs,
fetch_metrics,
fetch_router_domains,
)
from steward.core.alerts import MetricBatch
from steward.core.alerts import record_metric
metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"]
@@ -69,13 +57,6 @@ async def _do_scrape(app: "Quart") -> None:
except Exception:
logger.exception("Traefik scrape failed (url=%s)", metrics_url)
return
cfg = app.config["PLUGINS"]["traefik"]
try:
_router_domains = await fetch_router_domains(api_base(metrics_url, cfg.get("api_url") or ""))
except Exception as exc:
if _router_domains is not None:
logger.warning("Traefik API stopped answering (%s); certificates shown without routers", exc)
_router_domains = None
now = time.monotonic()
elapsed = now - _prev_time if _prev_time else 60.0
@@ -90,7 +71,6 @@ async def _do_scrape(app: "Quart") -> None:
async with app.db_sessionmaker() as session:
async with session.begin():
batch = MetricBatch()
# ── Per-service metrics ───────────────────────────────────────────
for router_name, metrics in router_metrics.items():
row = TraefikMetric(
@@ -115,14 +95,12 @@ async def _do_scrape(app: "Quart") -> None:
("latency_p99_ms", metrics["latency_p99_ms"]),
("response_bytes_rate", metrics["response_bytes_rate"]),
]:
# Stamped with the scrape, so a router's latest figures
# (routes.latest_per_router) all carry the same moment.
batch.add(
await record_metric(
session=session,
source_module="traefik",
resource_name=router_name,
metric_name=metric_name,
value=value,
recorded_at=scraped_at,
)
# ── Global stats ──────────────────────────────────────────────────
@@ -147,15 +125,14 @@ async def _do_scrape(app: "Quart") -> None:
# Emit cert_expiry_days for alert rules
days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0
batch.add(
await record_metric(
session=session,
source_module="traefik",
resource_name=cert["cn"] or cert["serial"],
metric_name="cert_expiry_days",
value=max(0.0, days_remaining),
)
await batch.record(session)
logger.debug(
"Traefik scrape: %d router(s), %d cert(s)",
len(router_metrics),
plugins/traefik/scraper.py
+6 -76
View File
@@ -152,21 +152,21 @@ def compute_router_metrics(
def compute_global_stats(
current: ParsedMetrics,
previous: ParsedMetrics | None,
) -> dict[str, float | datetime | None]:
) -> dict[str, float | None]:
"""Extract process-level and config health metrics from a scrape.
Returns:
{
"open_conns_total": float | None,
"config_reloads_total": float | None,
"config_last_reload_at": datetime | None, # last successful reload
"config_last_reload_success": float | None, # 1.0 = success, 0.0 = failure
"process_memory_bytes": float | None,
}
"""
stats: dict[str, float | datetime | None] = {
stats: dict[str, float | None] = {
"open_conns_total": None,
"config_reloads_total": None,
"config_last_reload_at": None,
"config_last_reload_success": None,
"process_memory_bytes": None,
}
@@ -180,15 +180,11 @@ def compute_global_stats(
if reloads:
stats["config_reloads_total"] = sum(reloads.values())
# The moment of the last successful reload, as a Unix timestamp: Traefik
# names it "last_reload_success", but it is a time, not a 0/1 flag. Read
# as a flag, every scrape said "Reload failed" (#5355).
# Last reload success gauge
last_success = current.get("traefik_config_last_reload_success", {})
if last_success:
# Usually a single sample with no labels
ts = next(iter(last_success.values()))
if ts > 0:
stats["config_last_reload_at"] = datetime.fromtimestamp(ts, timezone.utc)
stats["config_last_reload_success"] = next(iter(last_success.values()))
# Process RSS memory
mem = current.get("process_resident_memory_bytes", {})
@@ -278,69 +274,3 @@ async def fetch_metrics(metrics_url: str) -> ParsedMetrics:
response = await client.get(metrics_url)
response.raise_for_status()
return parse_prometheus(response.text)
# ──────────────────────────────────────────────────────────────────────────────
# Router domains (Traefik API)
# ──────────────────────────────────────────────────────────────────────────────
#
# The metrics name each router but not the hosts it answers for, so which
# certificate serves which router is only knowable from Traefik's API. Read
# once per scrape; when the API isn't reachable, routers simply have no
# domains and certificates are shown on their own (#5254).
_HOST_MATCHER = re.compile(r"Host(?:SNI)?\(([^)]*)\)")
_QUOTED = re.compile(r"[`\"']([^`\"']+)[`\"']")
def rule_hosts(rule: str) -> list[str]:
"""The host names a router rule matches: Host(`a`, `b`) and HostSNI(`c`)
give ["a", "b", "c"]; HostRegexp and path-only rules give nothing."""
hosts: list[str] = []
for args in _HOST_MATCHER.findall(rule or ""):
hosts.extend(h.strip().lower() for h in _QUOTED.findall(args) if h.strip() != "*")
return hosts
def api_base(metrics_url: str, api_url: str = "") -> str:
"""The Traefik API's base: the configured api_url, else /api on the
metrics URL's own host (the default "traefik" entrypoint serves both)."""
if api_url:
return api_url.rstrip("/")
parts = httpx.URL(metrics_url)
return str(parts.copy_with(path="/api", query=None, fragment=None)).rstrip("/")
async def fetch_router_domains(base: str) -> dict[str, list[str]]:
"""{router name: host names} for every enabled HTTP and TCP router."""
out: dict[str, list[str]] = {}
async with httpx.AsyncClient(timeout=10.0) as client:
for kind in ("http", "tcp"):
response = await client.get(f"{base}/{kind}/routers", params={"per_page": 1000})
if kind == "tcp" and response.status_code == 404:
continue
response.raise_for_status()
for r in response.json():
if r.get("status", "enabled") != "enabled" or not r.get("name"):
continue
hosts = rule_hosts(r.get("rule", ""))
if hosts:
out.setdefault(r["name"], []).extend(hosts)
return out
def cert_names(cn: str | None, sans: str | None) -> set[str]:
"""Every name a certificate covers: its CN and SANs, lower-cased."""
names = {n.strip().lower() for n in (sans or "").split(",") if n.strip()}
if cn:
names.add(cn.strip().lower())
return names
def covers(names: set[str], host: str) -> bool:
"""Whether a certificate with these names is valid for host, a wildcard
"*.example.com" covering one label: "a.example.com", not "a.b.example.com"."""
if host in names:
return True
head, _, rest = host.partition(".")
return bool(rest) and f"*.{rest}" in names
plugins/traefik/templates/traefik/index.html
+1 -6
View File
@@ -7,12 +7,7 @@
<h1 class="page-title" style="margin-bottom:0;">Traefik Services</h1>
<span style="color:var(--text-muted);font-size:0.85rem;">polling every {{ poll_interval }}s</span>
</div>
<div style="display:flex;align-items:center;gap:0.75rem;flex-wrap:wrap;">
{% if session.user_role == 'admin' %}
<a href="/settings/thresholds/#traefik" class="btn btn-ghost btn-sm">Thresholds</a>
{% endif %}
{% include "_time_range.html" %}
</div>
{% include "_time_range.html" %}
</div>
<div id="traefik-rows"
plugins/traefik/templates/traefik/rows.html
+106 -121
View File
@@ -1,142 +1,127 @@
{# traefik/rows.html — HTMX fragment for the Traefik page: routers that need
attention first (routes.router_state), the proxy's own counts, the healthy
routers folded into one table, then certificates by soonest expiry. #}
{% from "_macros.html" import stat_tile, status_badge %}
{# plugins/traefik/templates/traefik/rows.html #}
{# ── Counts ───────────────────────────────────────────────────────────────── #}
<div class="stat-strip">
{{ stat_tile("Needs attention", problems|length, state="crit" if problems else "inactive") }}
{{ stat_tile("Healthy routers", healthy|length, state="up" if healthy else "inactive") }}
{% if global_stat %}
{% if global_stat.config_last_reload_at %}
{{ stat_tile("Config reloaded", global_stat.config_last_reload_at|ago,
hint=(global_stat.config_reloads_total|int ~ " reloads") if global_stat.config_reloads_total is not none else none) }}
{# ── Global stats bar ──────────────────────────────────────────────────────── #}
{% if global_stat %}
<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--card-bg);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;">
{% if global_stat.open_conns_total is not none %}
<span>
<span style="color:var(--text-muted);">Open conns</span>
<span style="margin-left:0.4rem;font-weight:600;">{{ global_stat.open_conns_total | int }}</span>
</span>
{% endif %}
{% if global_stat.open_conns_total is not none %}{{ stat_tile("Open connections", global_stat.open_conns_total|int) }}{% endif %}
{% if global_stat.process_memory_bytes is not none %}{{ stat_tile("Memory", global_stat.process_memory_bytes|bytes) }}{% endif %}
{% if global_stat.config_last_reload_success is not none %}
<span>
<span style="color:var(--text-muted);">Config</span>
{% if global_stat.config_last_reload_success == 1.0 %}
<span style="margin-left:0.4rem;color:var(--green);">&#10003; OK</span>
{% else %}
<span style="margin-left:0.4rem;color:var(--red);">&#10007; failed</span>
{% endif %}
{% if global_stat.config_reloads_total is not none %}
<span style="color:var(--text-dim);margin-left:0.3rem;">({{ global_stat.config_reloads_total | int }} reloads)</span>
{% endif %}
</span>
{% endif %}
{% if global_stat.process_memory_bytes is not none %}
<span>
<span style="color:var(--text-muted);">Memory</span>
{% set mb = (global_stat.process_memory_bytes / 1048576) %}
<span style="margin-left:0.4rem;">
{% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.0f"|format(mb) }} MB{% endif %}
</span>
</span>
{% endif %}
<span style="margin-left:auto;color:var(--text-dim);font-size:0.78rem;">
{{ global_stat.scraped_at.strftime("%H:%M:%S") }} UTC
</span>
</div>
{% endif %}
{% macro router_card(r) %}
<div class="card">
<div class="host-health-head">
{{ status_badge(r.state, "Failing" if r.state == 'crit' else "Warning") }}
<h3 class="section-title" style="margin:0;word-break:break-all;">{{ r.name }}</h3>
</div>
<p class="problem-detail" style="margin-bottom:0.5rem;">{{ r.why }}</p>
{# ── Per-service cards + cert card in shared columns layout ───────────────── #}
{% if router_data or certs %}
<div style="column-width:340px;column-count:3;column-gap:1rem;">
{% for r in router_data %}
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
<h3 class="section-title" style="margin-bottom:0.75rem;word-break:break-all;">{{ r.name }}</h3>
<table class="table" style="font-size:0.85rem;">
<tr><td class="field-hint">Req/s</td><td>{{ "%.2f"|format(r.latest.request_rate) }}</td><td>{{ r.sparkline_req|safe }}</td></tr>
<tr><td class="field-hint">Bandwidth</td><td>{{ r.latest.response_bytes_rate|rate }}</td><td>{{ r.sparkline_bytes|safe }}</td></tr>
<tr><td class="field-hint">p95</td><td class="tone-{{ status_info(status_class(r.latest.latency_p95_ms, 'latency'))[0] }}">{{ "%.0f"|format(r.latest.latency_p95_ms) }} ms</td><td>{{ r.sparkline_p95|safe }}</td></tr>
<tr><td class="field-hint">p99</td><td>{{ "%.0f"|format(r.latest.latency_p99_ms) }} ms</td><td></td></tr>
<tr><td class="field-hint">4xx</td><td class="{{ 'tone-warn' if r.latest.error_rate_4xx_pct > error_4xx else '' }}">{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%</td><td>{{ r.sparkline_4xx|safe }}</td></tr>
<tr><td class="field-hint">5xx</td><td class="{{ 'tone-crit' if r.latest.error_rate_5xx_pct > error_5xx else '' }}">{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%</td><td>{{ r.sparkline_5xx|safe }}</td></tr>
<tr>
<td style="color:var(--text-muted);padding:0.2rem 0;">Req/s</td>
<td style="padding:0.2rem 0.5rem;">{{ "%.2f"|format(r.latest.request_rate) }}</td>
<td style="padding:0.2rem 0;">{{ r.sparkline_req | safe }}</td>
</tr>
<tr>
<td style="color:var(--text-muted);padding:0.2rem 0;">Bandwidth</td>
<td style="padding:0.2rem 0.5rem;">
{% set bps = r.latest.response_bytes_rate %}
{% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
{% elif bps >= 1024 %}{{ "%.1f"|format(bps / 1024) }} KB/s
{% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
</td>
<td style="padding:0.2rem 0;"></td>
</tr>
<tr>
<td style="color:var(--text-muted);padding:0.2rem 0;">p95 ms</td>
<td style="padding:0.2rem 0.5rem;">{{ "%.1f"|format(r.latest.latency_p95_ms) }}</td>
<td style="padding:0.2rem 0;">{{ r.sparkline_p95 | safe }}</td>
</tr>
<tr>
<td style="color:var(--text-muted);padding:0.2rem 0;">p99 ms</td>
<td style="padding:0.2rem 0.5rem;">{{ "%.1f"|format(r.latest.latency_p99_ms) }}</td>
<td style="padding:0.2rem 0;"></td>
</tr>
<tr>
<td style="color:var(--text-muted);padding:0.2rem 0;">4xx %</td>
<td style="color:{% if r.latest.error_rate_4xx_pct > 1 %}var(--yellow){% else %}var(--green){% endif %};padding:0.2rem 0.5rem;">
{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%
</td>
<td style="padding:0.2rem 0;"></td>
</tr>
<tr>
<td style="color:var(--text-muted);padding:0.2rem 0;">5xx %</td>
<td style="color:{% if r.latest.error_rate_5xx_pct > 0.1 %}var(--red){% else %}var(--green){% endif %};padding:0.2rem 0.5rem;">
{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%
</td>
<td style="padding:0.2rem 0;">{{ r.sparkline_5xx | safe }}</td>
</tr>
</table>
<div class="field-hint" style="margin-top:0.5rem;">Last scraped {{ r.latest.scraped_at|ago }}</div>
<div style="color:var(--text-dim);font-size:0.75rem;margin-top:0.5rem;">
Last scraped {{ r.latest.scraped_at.strftime("%H:%M:%S") }} UTC
</div>
</div>
{% endmacro %}
{% endfor %}
{% if problems or healthy or gone or certs or old_certs %}
{# ── Routers that need attention ─────────────────────────────────────────── #}
{% if problems %}
{# A grid, not CSS columns: a column box drops its last card's bottom margin,
which left the cards touching the section below (#5214). #}
<div class="node-grid" style="margin-bottom:1rem;">
{% for r in problems %}{{ router_card(r) }}{% endfor %}
</div>
{% endif %}
{# ── Healthy routers, folded ─────────────────────────────────────────────── #}
{% if healthy %}
<details class="card" id="traefik-healthy" style="margin-bottom:1rem;">
<summary class="section-title" style="cursor:pointer;">{{ healthy|length }} healthy router{{ 's' if healthy|length != 1 }}</summary>
<div class="table-wrap" style="margin-top:0.75rem;">
<table class="table" style="font-size:0.85rem;">
<thead><tr><th>Router</th><th>Req/s</th><th>Bandwidth</th><th>p95</th><th>4xx</th><th>5xx</th></tr></thead>
<tbody>
{% for r in healthy %}
<tr>
<td style="word-break:break-all;">{{ r.name }}</td>
<td>{{ "%.2f"|format(r.latest.request_rate) }}</td>
<td>{{ r.latest.response_bytes_rate|rate }}</td>
<td class="tone-{{ status_info(status_class(r.latest.latency_p95_ms, 'latency'))[0] }}">{{ "%.0f"|format(r.latest.latency_p95_ms) }} ms</td>
<td>{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%</td>
<td>{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</details>
{% endif %}
{# ── Routers Traefik no longer reports: their last numbers are history, not a
state, so they are listed by when they were last seen and nothing else. ── #}
{% if gone %}
<details class="card" style="margin-bottom:1rem;">
<summary class="section-title" style="cursor:pointer;">{{ gone|length }} router{{ 's' if gone|length != 1 }} no longer reported</summary>
<p class="field-hint" style="margin:0.5rem 0;">Traefik stopped reporting these, usually because the service behind them was removed.</p>
<ul class="problem-list">
{% for r in gone %}
<li><span style="word-break:break-all;">{{ r.name }}</span> <span class="problem-detail">last seen {{ r.latest.scraped_at|ago }}</span></li>
{% endfor %}
</ul>
</details>
{% endif %}
{% if healthy %}
<script>
// The fragment is re-polled; keep the healthy list as the reader left it.
(function () {
var el = document.getElementById("traefik-healthy");
if (!el) return;
el.open = !!window.__traefikHealthyOpen;
el.addEventListener("toggle", function () { window.__traefikHealthyOpen = el.open; });
})();
</script>
{% endif %}
{# ── Certificates Traefik still serves, soonest expiry first; then the ones
it no longer reports (renewed or retired), kept here but off the
dashboard (#5254). #}
{% if certs or old_certs %}
<div class="card">
<div class="section-title" style="margin-bottom:0.75rem;">TLS certificates</div>
{% if certs %}
<table class="table" style="font-size:0.85rem;">
<thead><tr><th>Name</th><th>Routers</th><th>Expires</th><th style="text-align:right;">Left</th></tr></thead>
<tbody>
{% if certs %}
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
<div class="section-title" style="margin-bottom:0.75rem;">TLS Certificates</div>
<table class="table" style="font-size:0.83rem;width:100%;">
{% for cert in certs %}
<tr>
<td style="word-break:break-all;">{{ cert.cn }}{% if cert.sans %}<div class="field-hint">also {{ cert.sans|join(", ") }}</div>{% endif %}</td>
<td class="field-hint">{% if cert.routers is none %}—{% elif cert.routers %}{{ cert.routers|join(", ") }}{% else %}not in use{% endif %}</td>
<td class="field-hint">{{ cert.not_after.strftime("%Y-%m-%d") }}</td>
<td style="text-align:right;white-space:nowrap;" class="tone-{{ status_info(status_class(cert.days_remaining, 'tls_days'))[0] }}">{{ cert_left(cert.days_remaining) }}</td>
<td style="padding:0.2rem 0 0.2rem 0;word-break:break-all;">{{ cert.cn }}</td>
<td style="text-align:right;padding:0.2rem 0;white-space:nowrap;font-weight:600;
{% if cert.days_remaining < 7 %}color:var(--red);
{% elif cert.days_remaining < 30 %}color:var(--yellow);
{% else %}color:var(--green);{% endif %}">
{{ cert.days_remaining | int }}d
</td>
</tr>
<tr>
<td colspan="2" style="padding:0 0 0.4rem 0;color:var(--text-dim);font-size:0.75rem;">
{{ cert.not_after.strftime("%Y-%m-%d") }}
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
{% if old_certs %}
<details style="margin-top:0.75rem;">
<summary class="field-hint">No longer served ({{ old_certs|length }}) — renewed or retired; Traefik stopped reporting them</summary>
<table class="table" style="font-size:0.85rem;margin-top:0.5rem;">
<thead><tr><th>Name</th><th>Expires</th></tr></thead>
<tbody>
{% for cert in old_certs %}
<tr style="color:var(--text-muted);">
<td style="word-break:break-all;">{{ cert.cn }}</td>
<td class="field-hint">{{ cert.not_after.strftime("%Y-%m-%d") }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</details>
{% endif %}
</div>
{% endif %}
</div>
{% else %}
<div class="card">
<p class="field-hint">No Traefik metrics yet. Configure <code>metrics_url</code> in <code>config.yaml</code> under <code>plugins.traefik</code>.</p>
<p style="color:#606080;">No Traefik metrics yet. Configure <code>metrics_url</code> in <code>config.yaml</code> under <code>plugins.traefik</code>.</p>
</div>
{% endif %}
plugins/traefik/templates/traefik/traffic.html
+8 -11
View File
@@ -2,25 +2,26 @@
{% if summary %}
{# ── Overview bar ─────────────────────────────────────────────────────────── #}
<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--bg-card);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;align-items:center;">
<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--card-bg);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;align-items:center;">
<span>
<span style="color:var(--text-muted);">Total</span>
<span style="margin-left:0.4rem;font-weight:500;">{{ summary.total }}</span>
<span style="margin-left:0.4rem;font-weight:600;">{{ summary.total }}</span>
</span>
<span>
<span style="color:var(--text-muted);">Internal</span>
<span style="margin-left:0.4rem;color:var(--green);font-weight:500;">{{ summary.internal }}</span>
<span style="margin-left:0.4rem;color:var(--green);font-weight:600;">{{ summary.internal }}</span>
</span>
<span>
<span style="color:var(--text-muted);">External</span>
<span style="margin-left:0.4rem;color:{% if summary.external_pct > 50 %}var(--yellow){% else %}var(--green){% endif %};font-weight:500;">
<span style="margin-left:0.4rem;color:{% if summary.external_pct > 50 %}var(--yellow){% else %}var(--green){% endif %};font-weight:600;">
{{ summary.external }} ({{ "%.1f"|format(summary.external_pct) }}%)
</span>
</span>
<span>
<span style="color:var(--text-muted);">Bytes</span>
<span style="margin-left:0.4rem;">
{{ summary.total_bytes|bytes }}
{% set mb = summary.total_bytes / 1048576 %}
{% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.1f"|format(mb) }} MB{% endif %}
</span>
</span>
@@ -92,13 +93,9 @@
{% else %}
<div class="card" style="color:var(--text-muted);font-size:0.85rem;">
No access log data yet.
{% if not access_log_enabled %}
The access log is off. Turn it on in <a href="/settings/plugins/traefik/">Settings → Plugins → Traefik</a>,
and mount Traefik's log directory into Steward so it can read the file.
{% else %}
No requests in the last {{ range_key }}. The access log is on; if this stays empty, check that
{% if access_log_path %}<code>{{ access_log_path }}</code>{% else %}the log path{% endif %} is mounted into Steward
and that Traefik is writing to it.
Enable <code>access_log.enabled: true</code> in your plugin config and bind-mount the Traefik log directory.
{% endif %}
</div>
{% endif %}
plugins/traefik/templates/traefik/widget.html
+52 -51
View File
@@ -1,55 +1,56 @@
{# plugins/traefik/templates/traefik/widget.html — the dashboard Routers widget
(#5254): a verdict, routers in trouble with why, then the busiest healthy
ones, under one header. Only routers Traefik still reports. A certificate
shows on the router it serves (TLS column, days left); when Traefik's API
can't say which hosts a router serves, the expiring certificates are listed
after the routers instead. #}
{% from "_macros.html" import status_dot %}
{% if not total %}
<div class="w-verdict" style="color:var(--text-muted);">No Traefik data yet.</div>
{% else %}
{% set crit = problems | selectattr("state", "equalto", "crit") | list %}
<div class="w-verdict {{ 'tone-crit' if crit else ('tone-warn' if problems or loose_certs else 'tone-ok') }}">
{%- if problems|length == 1 %}{{ problems[0].name.split('@')[0] }}: {{ problems[0].why }}.
{%- elif problems %}{{ problems|length }} of {{ total }} routers need a look.
{%- else %}All {{ total }} router{{ 's' if total != 1 }} healthy{% if loose_certs %}; {{ loose_certs|length }} certificate{{ 's' if loose_certs|length != 1 }} expiring{% endif %}.{% endif -%}
</div>
<table class="w-table">
<colgroup><col><col style="width:4rem"><col class="w-wide-only" style="width:5.5rem"><col style="width:4.2rem">{% if domains_known %}<col style="width:4.5rem">{% endif %}</colgroup>
<thead><tr><th>Router</th><th>Req/s</th><th class="w-wide-only">Bandwidth</th><th>p95</th>{% if domains_known %}<th>TLS</th>{% endif %}</tr></thead>
<tbody>
{% for r in problems + healthy %}
{% set m = r.latest %}
<tr>
<td class="w-name" title="{{ r.name }}">
{{- status_dot(r.state) -}}
{{ r.name.split('@')[0] }}
{%- if r.why %}<span class="w-sub tone-{{ r.state }}">{{ r.why }}</span>{% endif %}
</td>
<td>{% if m.request_rate is none %}—{% else %}{{ "%.2f"|format(m.request_rate) }}{% endif %}</td>
<td class="w-wide-only">{% if m.response_bytes_rate is none %}—{% else %}{{ m.response_bytes_rate|rate }}{% endif %}</td>
<td style="{{ threshold_style(m.latency_p95_ms, 'latency') if m.latency_p95_ms is not none }}">
{%- if m.latency_p95_ms is none %}—{% else %}{{ "%.0f"|format(m.latency_p95_ms) }} ms{% endif -%}
</td>
{% if domains_known %}
<td {% if r.cert %}style="{{ threshold_style(r.cert.days, 'tls_days') }}" title="Certificate {{ r.cert.cn }}"{% endif %}>
{%- if r.cert %}{{ cert_left(r.cert.days) }}{% else %}—{% endif -%}
</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
{% if healthy_more %}<div class="w-more">+{{ healthy_more }} more healthy{% if session.user_id %} · <a href="/plugins/traefik/">all routers</a>{% endif %}</div>{% endif %}
{% if loose_certs %}
<div class="w-section">Certificates expiring</div>
{% for c in loose_certs %}
<div class="w-item">
{{- status_dot(status_class(c.days, 'tls_days')) -}}
<span class="w-label" title="{{ c.cn }}">{{ c.cn }}</span>
<span class="w-meta" style="{{ threshold_style(c.days, 'tls_days') }}">{{ cert_left(c.days) }}</span>
{# plugins/traefik/templates/traefik/widget.html #}
{% if expiring_certs %}
<div style="margin-bottom:0.6rem;">
{% for cert in expiring_certs %}
<div style="display:flex;justify-content:space-between;font-size:0.8rem;padding:0.15rem 0;
{% if cert.days_remaining < 7 %}color:var(--red);{% else %}color:var(--yellow);{% endif %}">
<span>&#9888; {{ cert.cn }}</span>
<span>{{ cert.days_remaining | int }}d</span>
</div>
{% endfor %}
<div class="w-note">Traefik's API didn't say which hosts each router serves, so these can't be shown on their routers.</div>
</div>
{% endif %}
{% if router_data %}
{% for r in router_data %}
<div class="ping-row">
<div class="ping-meta">
<span class="ping-name" style="font-size:0.8rem;word-break:break-all;">{{ r.name }}</span>
</div>
<div style="flex:1;display:flex;gap:1.5rem;font-size:0.82rem;font-variant-numeric:tabular-nums;">
<span title="Requests/s">
<span style="color:var(--text-muted);">req/s</span>
<span style="margin-left:0.3rem;">{{ "%.2f"|format(r.latest.request_rate) }}</span>
</span>
<span title="Bandwidth">
<span style="color:var(--text-muted);">bw</span>
<span style="margin-left:0.3rem;">
{% set bps = r.latest.response_bytes_rate %}
{% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
{% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
{% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
</span>
</span>
<span title="p95 latency">
<span style="color:var(--text-muted);">p95</span>
<span style="margin-left:0.3rem;
{% if r.latest.latency_p95_ms > 500 %}color:var(--red){% elif r.latest.latency_p95_ms > 200 %}color:var(--yellow){% else %}color:var(--green){% endif %};">
{{ "%.0f"|format(r.latest.latency_p95_ms) }}ms
</span>
</span>
{% if r.latest.error_rate_5xx_pct > 0 %}
<span title="5xx error rate">
<span style="color:var(--red);">{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% 5xx</span>
</span>
{% endif %}
</div>
</div>
{% endfor %}
{% if total_routers > 5 %}
<div style="color:var(--text-dim);font-size:0.75rem;padding:0.25rem 0;">
+{{ total_routers - 5 }} more — <a href="/plugins/traefik/">view all →</a>
</div>
{% endif %}
{% else %}
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No Traefik data yet.</p>
{% endif %}
plugins/traefik/templates/traefik/widget_access_log.html
+2 -2
View File
@@ -5,11 +5,11 @@
<div style="display:flex;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
<div>
<span style="font-size:1.4rem;font-weight:500;line-height:1;">{{ summary.total | int }}</span>
<span style="font-size:1.4rem;font-weight:700;line-height:1;">{{ summary.total | int }}</span>
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">total req</span>
</div>
<div>
<span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--orange);">
<span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--orange);">
{{ "%.1f" | format(summary.external_pct) }}%
</span>
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">external</span>
plugins/traefik/templates/traefik/widget_request_chart.html
+63 -88
View File
@@ -1,100 +1,75 @@
{# traefik/widget_request_chart.html — requests per second across all routers
and the share answered 5xx, over the last few hours (#5256). The verdict
carries the figures, so the widget still says something if the chart
can't draw; a chart that fails says why in place of the canvas. #}
{# traefik/widget_request_chart.html — Chart.js request rate + error % over time #}
{% if not labels_json or labels_json == '[]' %}
<div class="w-verdict" style="color:var(--text-muted);">No Traefik requests recorded in the last {{ hours }}h.</div>
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No data for last {{ hours }}h.</div>
{% else %}
<div class="w-verdict {{ 'tone-crit' if peak_err and peak_err >= 1 else ('tone-warn' if peak_err else 'tone-ok') }}">
Now {{ "%.2f"|format(now_req) }} req/s across all routers, peak {{ "%.2f"|format(peak_req) }}.
{%- if peak_err %} 5xx peaked at {{ "%.2f"|format(peak_err) }}% of responses.{% else %} No 5xx.{% endif %}
</div>
<div style="position:relative;height:160px;" id="chart-traefik-req-box-{{ widget_id }}">
<canvas id="chart-traefik-req-{{ widget_id }}" aria-label="Requests per second and 5xx share, last {{ hours }} hours"></canvas>
<div style="position:relative;height:160px;">
<canvas id="chart-traefik-req-{{ widget_id }}"></canvas>
</div>
<script>
(function() {
var box = document.getElementById("chart-traefik-req-box-{{ widget_id }}");
var ctx = document.getElementById("chart-traefik-req-{{ widget_id }}");
if (!ctx) return;
function fail(why) {
box.style.height = "auto";
box.innerHTML = '<div class="w-note"></div>';
box.firstChild.textContent = "The chart couldn't be drawn: " + why;
}
if (!window.Chart) { fail("the chart library didn't load."); return; }
try {
var existing = Chart.getChart(ctx);
if (existing) existing.destroy();
var times = {{ labels_json | safe }};
var hhmm = function (ms) {
return new Date(ms).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" });
};
var chart = new Chart(ctx.getContext("2d"), {
type: "line",
data: {
labels: times,
datasets: [
{
label: "Requests/s (all routers)",
data: {{ req_rate_json | safe }},
borderColor: stewardColor("--blue"),
backgroundColor: stewardColor("--blue-dim"),
fill: true, tension: 0.3, pointRadius: 0, borderWidth: 1.5, yAxisID: "y",
},
{
label: "5xx share %",
data: {{ error_rate_json | safe }},
borderColor: stewardColor("--red"),
fill: false, tension: 0.3, pointRadius: 0, borderWidth: 1.5, yAxisID: "y2",
},
]
},
options: {
responsive: true,
maintainAspectRatio: false,
animation: false,
interaction: { mode: "index", intersect: false },
plugins: {
legend: { labels: { boxWidth: 10, font: { size: 10 } } },
tooltip: { callbacks: { title: function (items) { return items.length ? hhmm(times[items[0].dataIndex]) : ""; } } },
if (!ctx || !window.Chart) return;
var existing = Chart.getChart(ctx);
if (existing) existing.destroy();
new Chart(ctx.getContext("2d"), {
type: "line",
data: {
labels: {{ labels_json | safe }},
datasets: [
{
label: "Req/s",
data: {{ req_rate_json | safe }},
borderColor: "#7c5ef4",
backgroundColor: "rgba(124,94,244,0.08)",
fill: true,
tension: 0.3,
pointRadius: 0,
borderWidth: 1.5,
yAxisID: "y",
},
scales: {
x: {
ticks: {
font: { size: 10 }, maxRotation: 0, autoSkip: true, maxTicksLimit: 6,
callback: function (value) { return hhmm(times[value]); },
},
grid: { display: false },
},
y: {
min: 0,
ticks: { font: { size: 10 } },
title: { display: true, text: "req/s", font: { size: 9 } },
},
y2: {
position: "right",
min: 0,
suggestedMax: 5,
grid: { drawOnChartArea: false },
ticks: { color: stewardColor("--red"), font: { size: 10 } },
title: { display: true, text: "5xx %", color: stewardColor("--red"), font: { size: 9 } },
},
{
label: "5xx %",
data: {{ error_rate_json | safe }},
borderColor: "#e83848",
fill: false,
tension: 0.3,
pointRadius: 0,
borderWidth: 1.5,
yAxisID: "y2",
},
]
},
options: {
responsive: true,
maintainAspectRatio: false,
animation: false,
plugins: {
legend: {
labels: { color: "#6858a0", boxWidth: 10, font: { size: 10 } }
}
},
scales: {
x: { display: false },
y: {
min: 0,
grid: { color: "rgba(30,30,62,0.8)" },
ticks: { color: "#6858a0", font: { size: 10 } },
title: { display: true, text: "req/s", color: "#6858a0", font: { size: 9 } },
},
y2: {
position: "right",
min: 0,
max: 100,
grid: { drawOnChartArea: false },
ticks: { color: "#e83848", font: { size: 10 } },
title: { display: true, text: "5xx%", color: "#e83848", font: { size: 9 } },
},
}
});
// The chart is built while the dashboard may still be placing this cell,
// and a chart sized then can stay blank until a window resize (#5214).
// Measure and draw again once layout has settled.
setTimeout(function () {
if (!ctx.isConnected) return;
chart.resize();
chart.update("none");
}, 250);
} catch (err) {
fail(err && err.message ? err.message : String(err));
}
}
});
})();
</script>
<div class="w-note" style="text-align:right;">last {{ hours }}h, your local time</div>
<div style="font-size:0.72rem;color:var(--text-muted);margin-top:0.35rem;text-align:right;">
last {{ hours }}h
</div>
{% endif %}
plugins/unifi/__init__.py
-14
View File
@@ -16,14 +16,6 @@ def setup(app: "Quart") -> None:
UnifiDevice, UnifiKnownClient, UnifiEvent, UnifiAlarm,
UnifiSpeedtestResult, UnifiNetwork, UnifiPortForward, UnifiFwRule,
)
from steward.core.cleanup import prune_by_age
# Samples and history older than data.retention_days are pruned by the
# cleanup (#5155). Devices, known clients and config tables are current state.
for column in (UnifiWanStat.scraped_at, UnifiClientSnapshot.scraped_at,
UnifiWlanStat.scraped_at, UnifiDpiSnapshot.scraped_at,
UnifiEvent.event_time, UnifiAlarm.alarm_time,
UnifiSpeedtestResult.run_at):
prune_by_age(column)
def get_scheduled_tasks() -> list:
@@ -39,9 +31,3 @@ def get_blueprint():
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (UniFi network overview).
return [{"label": "UniFi", "href": "/plugins/unifi/"}]
def get_alert_metrics() -> dict[str, list[str]]:
"""What scheduler.py records: the WAN (resource "wan") and the client count
(resource "clients")."""
return {"unifi": ["is_up", "latency_ms", "total_clients"]}
plugins/unifi/models.py
+5 -5
View File
@@ -1,10 +1,10 @@
# plugins/unifi/models.py
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from steward.models.base import Base
from steward.core.ids import time_ordered_id
# ── Time-series (one row per scrape) ──────────────────────────────────────────
@@ -13,7 +13,7 @@ class UnifiWanStat(Base):
"""WAN interface health — one row per scrape."""
__tablename__ = "unifi_wan_stats"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
wan_ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
@@ -27,7 +27,7 @@ class UnifiClientSnapshot(Base):
"""Aggregated client counts + top talkers — one row per scrape."""
__tablename__ = "unifi_client_snapshots"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
total_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
wireless_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
@@ -41,7 +41,7 @@ class UnifiWlanStat(Base):
"""Per-SSID/radio stats — one row per scrape per SSID+radio combination."""
__tablename__ = "unifi_wlan_stats"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
ssid: Mapped[str] = mapped_column(String(255), nullable=False)
radio: Mapped[str | None] = mapped_column(String(8), nullable=True) # ng=2.4GHz, na=5GHz, 6e=6GHz
@@ -56,7 +56,7 @@ class UnifiDpiSnapshot(Base):
"""DPI application/category traffic snapshot — one row per scrape (JSON blob)."""
__tablename__ = "unifi_dpi_snapshots"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
# JSON: [{"cat_id", "cat_name", "rx_bytes", "tx_bytes"}] aggregated across all clients
by_category_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
plugins/unifi/routes.py
+66 -58
View File
@@ -2,11 +2,11 @@
from __future__ import annotations
import json
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import func, select
from sqlalchemy import Integer, cast, func, select
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, time_bucket
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds
from .models import (
UnifiAlarm, UnifiClientSnapshot, UnifiDevice, UnifiDpiSnapshot,
UnifiEvent, UnifiFwRule, UnifiKnownClient, UnifiNetwork,
@@ -49,58 +49,6 @@ async def index():
)
# A UniFi device's `state` code, as (status_dot state, label). Only 1 is
# working; the rest are named rather than all drawn as "Down" (#5207).
DEVICE_STATES = {
0: ("down", "Offline"), 1: ("up", "Online"), 2: ("pending", "Pending adoption"),
3: ("degraded", "Firmware mismatch"), 4: ("pending", "Upgrading"), 5: ("pending", "Provisioning"),
6: ("degraded", "Heartbeat missed"), 7: ("pending", "Adopting"), 8: ("pending", "Deleting"),
9: ("down", "Inform error"), 10: ("down", "Adoption failed"), 11: ("degraded", "Isolated"),
}
_DEVICE_ORDER = {"down": 0, "degraded": 1, "pending": 2, "up": 3}
def device_state(code) -> tuple[str, str]:
return DEVICE_STATES.get(code, ("unknown", f"State {code}"))
def devices_in_order(devices) -> list:
"""Offline and failing devices first, then ones in a transition, then the
working ones; by name within each."""
return sorted(devices, key=lambda d: (_DEVICE_ORDER.get(device_state(d.state)[0], 1),
(d.name or d.mac or "").lower()))
async def history(db, since, b_secs: int):
"""(WAN rows, client-count rows) averaged per `b_secs` window since
`since`, oldest first, for the overview's sparklines."""
wan_bucket = time_bucket(UnifiWanStat.scraped_at, b_secs)
wan = (await db.execute(
select(
func.avg(UnifiWanStat.latency_ms).label("latency_ms"),
func.avg(UnifiWanStat.rx_bytes_rate).label("rx_bytes_rate"),
func.avg(UnifiWanStat.tx_bytes_rate).label("tx_bytes_rate"),
func.min(UnifiWanStat.scraped_at).label("scraped_at"),
wan_bucket,
)
.where(UnifiWanStat.scraped_at >= since)
.group_by(wan_bucket)
.order_by(wan_bucket)
)).all()
cs_bucket = time_bucket(UnifiClientSnapshot.scraped_at, b_secs)
clients = (await db.execute(
select(
func.avg(UnifiClientSnapshot.total_clients).label("total_clients"),
func.min(UnifiClientSnapshot.scraped_at).label("scraped_at"),
cs_bucket,
)
.where(UnifiClientSnapshot.scraped_at >= since)
.group_by(cs_bucket)
.order_by(cs_bucket)
)).all()
return wan, clients
@unifi_bp.get("/rows")
@require_role(UserRole.viewer)
async def rows():
@@ -110,7 +58,23 @@ async def rows():
b_secs = bucket_seconds(since)
async with current_app.db_sessionmaker() as db:
wan_history, client_history = await history(db, since, b_secs)
# WAN history — bucketed for accurate full-range sparklines
wan_bucket = (
cast(func.strftime('%s', UnifiWanStat.scraped_at), Integer) / b_secs
).label("bucket")
result = await db.execute(
select(
func.avg(UnifiWanStat.latency_ms).label("latency_ms"),
func.avg(UnifiWanStat.rx_bytes_rate).label("rx_bytes_rate"),
func.avg(UnifiWanStat.tx_bytes_rate).label("tx_bytes_rate"),
func.min(UnifiWanStat.scraped_at).label("scraped_at"),
wan_bucket,
)
.where(UnifiWanStat.scraped_at >= since)
.group_by(wan_bucket)
.order_by(wan_bucket)
)
wan_history = result.all()
# Latest WAN — always most recent raw row for current displayed values
result = await db.execute(
@@ -128,10 +92,25 @@ async def rows():
# All devices
result = await db.execute(
select(UnifiDevice)
select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name)
)
devices = devices_in_order(result.scalars().all())
devices = result.scalars().all()
# Client count history — bucketed
cs_bucket = (
cast(func.strftime('%s', UnifiClientSnapshot.scraped_at), Integer) / b_secs
).label("bucket")
result = await db.execute(
select(
func.avg(UnifiClientSnapshot.total_clients).label("total_clients"),
func.min(UnifiClientSnapshot.scraped_at).label("scraped_at"),
cs_bucket,
)
.where(UnifiClientSnapshot.scraped_at >= since)
.group_by(cs_bucket)
.order_by(cs_bucket)
)
client_history = result.all()
# Latest WLAN stats (always most recent per SSID)
result = await db.execute(
@@ -170,7 +149,6 @@ async def rows():
latest_wan=latest_wan,
latest_clients=latest_clients,
devices=devices,
device_state=device_state,
top_clients=top_clients,
wlan_latest=wlan_latest,
speedtest=speedtest,
@@ -346,3 +324,33 @@ async def widget_clients():
snapshot=snapshot,
widget_id=widget_id,
)
@unifi_bp.get("/widget/devices")
@require_role(UserRole.viewer)
async def widget_devices():
"""HTMX dashboard widget: infrastructure devices with optional type filter."""
type_filter = request.args.get("type_filter", "all")
widget_id = request.args.get("wid", "0")
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name)
)
all_devices = result.scalars().all()
if type_filter == "wireless":
devices = [d for d in all_devices if d.device_type == "uap"]
elif type_filter == "wired":
devices = [d for d in all_devices if d.device_type != "uap"]
elif type_filter == "offline":
devices = [d for d in all_devices if d.state != 1]
else:
devices = list(all_devices)
return await render_template(
"unifi/widget_devices.html",
devices=devices,
type_filter=type_filter,
widget_id=widget_id,
)
plugins/unifi/scheduler.py
+7 -6
View File
@@ -54,7 +54,7 @@ async def _do_poll(app: "Quart") -> None:
from .client import UnifiClient
from .models import UnifiClientSnapshot, UnifiDevice, UnifiWanStat
from steward.core.alerts import MetricBatch
from steward.core.alerts import record_metric
cfg = app.config["PLUGINS"]["unifi"]
@@ -171,27 +171,28 @@ async def _do_poll(app: "Quart") -> None:
))
# ── Alert metrics ─────────────────────────────────────────────────
batch = MetricBatch()
if wan_stat.latency_ms is not None:
batch.add(
await record_metric(
session=session,
source_module="unifi",
resource_name="wan",
metric_name="latency_ms",
value=wan_stat.latency_ms,
)
batch.add(
await record_metric(
session=session,
source_module="unifi",
resource_name="wan",
metric_name="is_up",
value=1.0 if wan_stat.is_up else 0.0,
)
batch.add(
await record_metric(
session=session,
source_module="unifi",
resource_name="clients",
metric_name="total_clients",
value=float(len(clients)),
)
await batch.record(session)
logger.debug(
"UniFi poll: wan=%s, clients=%d, devices=%d",
plugins/unifi/templates/unifi/dpi.html
+9 -3
View File
@@ -11,7 +11,7 @@
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
<div class="section-title" style="margin-bottom:0.75rem;">
Traffic by Category
<span style="float:right;font-weight:normal;font-size:0.78rem;color:var(--text-dim);">as of {{ snapshot.scraped_at|ago }}</span>
<span style="float:right;font-weight:normal;font-size:0.78rem;color:var(--text-dim);">{{ snapshot.scraped_at.strftime("%H:%M") }}</span>
</div>
{% set total_bytes = categories | sum(attribute='rx_bytes') + categories | sum(attribute='tx_bytes') %}
<table class="table" style="width:100%;font-size:0.82rem;">
@@ -21,7 +21,10 @@
<tr>
<td style="padding:0.25rem 0.5rem 0.25rem 0;">{{ cat.cat_name or cat.cat_id }}</td>
<td style="padding:0.25rem 0.3rem;text-align:right;color:var(--text-muted);font-size:0.78rem;white-space:nowrap;">
{{ bytes|bytes }}
{% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB
{% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB
{% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB
{% else %}{{ bytes }} B{% endif %}
</td>
<td style="padding:0.25rem 0;width:4rem;">
<div style="background:var(--border);border-radius:2px;height:6px;overflow:hidden;">
@@ -47,7 +50,10 @@
</td>
<td style="padding:0.25rem 0;text-align:right;font-size:0.78rem;color:var(--text-muted);white-space:nowrap;">
{% set bytes = c.bytes %}
{{ bytes|bytes }}
{% if bytes >= 1073741824 %}{{ "%.1f"|format(bytes / 1073741824) }} GB
{% elif bytes >= 1048576 %}{{ "%.1f"|format(bytes / 1048576) }} MB
{% elif bytes >= 1024 %}{{ "%.0f"|format(bytes / 1024) }} KB
{% else %}{{ bytes }} B{% endif %}
</td>
</tr>
{% endfor %}
plugins/unifi/templates/unifi/events.html
+9 -3
View File
@@ -23,12 +23,18 @@
<th style="text-align:left;font-weight:normal;padding:0 0 0.4rem;">Message</th>
</tr>
{% for ev in events %}
<tr style="{% if loop.index is divisibleby 2 %}background:var(--fs-table-stripe);{% endif %}">
<tr style="{% if loop.index is divisibleby 2 %}background:var(--row-alt,rgba(255,255,255,0.02));{% endif %}">
<td style="padding:0.25rem 0.5rem 0.25rem 0;color:var(--text-dim);font-size:0.77rem;white-space:nowrap;">
{{ ev.event_time|ago }}
{{ ev.event_time.strftime("%Y-%m-%d %H:%M:%S") }}
</td>
<td style="padding:0.25rem 0.5rem;">
<span class="badge badge-{{ {'wlan': 'info', 'wan': 'ok', 'ids': 'crit'}.get(ev.category, 'dim') }}">{{ ev.category }}</span>
<span style="font-size:0.75rem;padding:0.1rem 0.4rem;border-radius:3px;background:
{% if ev.category == 'wlan' %}rgba(96,96,192,0.25)
{% elif ev.category == 'wan' %}rgba(96,160,96,0.25)
{% elif ev.category == 'ids' %}rgba(192,64,64,0.25)
{% else %}rgba(128,128,128,0.15){% endif %};">
{{ ev.category }}
</span>
</td>
<td style="padding:0.25rem 0.5rem;color:var(--text-dim);font-family:monospace;font-size:0.75rem;">
{{ ev.event_key.replace("EVT_", "") if ev.event_key else "" }}
plugins/unifi/templates/unifi/inventory.html
+8 -3
View File
@@ -61,7 +61,7 @@
{% if rule.ruleset != ns.current_rs %}
{% set ns.current_rs = rule.ruleset %}
<tr>
<td colspan="3" style="padding:0.4rem 0 0.2rem;color:var(--text-muted);font-size:0.75rem;font-weight:500;letter-spacing:0.04em;text-transform:uppercase;">
<td colspan="3" style="padding:0.4rem 0 0.2rem;color:var(--text-muted);font-size:0.75rem;font-weight:600;letter-spacing:0.04em;text-transform:uppercase;">
{{ rule.ruleset }}
</td>
</tr>
@@ -73,7 +73,12 @@
</span>
</td>
<td style="padding:0.2rem 0.3rem;white-space:nowrap;">
<span class="badge badge-{{ {'accept': 'ok', 'drop': 'crit'}.get(rule.action, 'warn') }}">{{ rule.action }}</span>
<span style="font-size:0.75rem;padding:0.1rem 0.35rem;border-radius:3px;background:
{% if rule.action == 'accept' %}rgba(64,160,64,0.2)
{% elif rule.action == 'drop' %}rgba(192,64,64,0.2)
{% else %}rgba(192,128,64,0.2){% endif %};">
{{ rule.action }}
</span>
</td>
<td style="padding:0.2rem 0;color:var(--text-dim);font-size:0.76rem;">
{% if rule.protocol %}{{ rule.protocol }}{% endif %}
@@ -117,7 +122,7 @@
{{ kc.ip or "—" }}
</td>
<td style="padding:0.2rem 0;text-align:right;color:var(--text-dim);font-size:0.75rem;white-space:nowrap;">
{% if kc.last_seen %}{{ kc.last_seen|ago }}{% else %}—{% endif %}
{% if kc.last_seen %}{{ kc.last_seen.strftime("%m-%d %H:%M") }}{% else %}—{% endif %}
</td>
</tr>
{% endfor %}
plugins/unifi/templates/unifi/rows.html
+67 -29
View File
@@ -1,20 +1,57 @@
{# plugins/unifi/templates/unifi/rows.html #}
{% from "_macros.html" import stat_tile, status_badge, status_dot %}
{# ── WAN health ───────────────────────────────────────────────────────────── #}
{# ── WAN health bar ───────────────────────────────────────────────────────── #}
{% if latest_wan %}
<div class="stat-strip">
{{ stat_tile("WAN", "Up" if latest_wan.is_up else "Down", state="up" if latest_wan.is_up else "down",
hint=latest_wan.wan_ip) }}
<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--card-bg);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;align-items:center;">
<span>
<span style="color:var(--text-muted);">WAN</span>
<span style="margin-left:0.4rem;font-weight:600;color:{% if latest_wan.is_up %}var(--green){% else %}var(--red){% endif %};">
{% if latest_wan.is_up %}&#10003; Up{% else %}&#10007; Down{% endif %}
</span>
{% if latest_wan.wan_ip %}
<span style="color:var(--text-dim);margin-left:0.4rem;font-family:monospace;font-size:0.8rem;">{{ latest_wan.wan_ip }}</span>
{% endif %}
</span>
{% if latest_wan.latency_ms is not none %}
{{ stat_tile("Latency", "%.0f ms"|format(latest_wan.latency_ms), state=status_class(latest_wan.latency_ms, 'latency'),
hint=sparkline_latency|safe) }}
<span>
<span style="color:var(--text-muted);">Latency</span>
<span style="margin-left:0.4rem;
{% if latest_wan.latency_ms > 100 %}color:var(--red){% elif latest_wan.latency_ms > 50 %}color:var(--yellow){% else %}color:var(--green){% endif %};">
{{ "%.1f"|format(latest_wan.latency_ms) }}ms
</span>
<span style="margin-left:0.3rem;">{{ sparkline_latency | safe }}</span>
</span>
{% endif %}
{% if latest_wan.rx_bytes_rate is not none %}
{{ stat_tile("Download", latest_wan.rx_bytes_rate|rate, hint=sparkline_rx|safe) }}
{{ stat_tile("Upload", latest_wan.tx_bytes_rate|rate, hint=sparkline_tx|safe) }}
<span>
<span style="color:var(--text-muted);">&#8595; RX</span>
<span style="margin-left:0.3rem;">
{% set bps = latest_wan.rx_bytes_rate %}
{% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
{% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s
{% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
</span>
<span style="margin-left:0.3rem;">{{ sparkline_rx | safe }}</span>
</span>
<span>
<span style="color:var(--text-muted);">&#8593; TX</span>
<span style="margin-left:0.3rem;">
{% set bps = latest_wan.tx_bytes_rate %}
{% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
{% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }} KB/s
{% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
</span>
<span style="margin-left:0.3rem;">{{ sparkline_tx | safe }}</span>
</span>
{% endif %}
{% if latest_wan.uptime_seconds is not none %}
<span style="color:var(--text-dim);font-size:0.8rem;">
up {{ (latest_wan.uptime_seconds // 3600) }}h {{ ((latest_wan.uptime_seconds % 3600) // 60) }}m
</span>
{% endif %}
{% if latest_wan.uptime_seconds is not none %}{{ stat_tile("WAN uptime", latest_wan.uptime_seconds|duration) }}{% endif %}
</div>
{% endif %}
@@ -30,7 +67,7 @@
</div>
<div style="display:flex;gap:1.5rem;font-size:0.9rem;margin-bottom:0.75rem;font-variant-numeric:tabular-nums;">
<span>
<span style="font-size:1.4rem;color:var(--text);">{{ latest_clients.total_clients }}</span>
<span style="font-size:1.4rem;font-weight:700;color:var(--text);">{{ latest_clients.total_clients }}</span>
<span style="color:var(--text-muted);font-size:0.8rem;margin-left:0.2rem;">total</span>
</span>
<span style="color:var(--text-muted);font-size:0.85rem;align-self:flex-end;">
@@ -45,12 +82,12 @@
{# Infrastructure devices card #}
{% if devices %}
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
<div class="section-title" style="margin-bottom:0.75rem;">Infrastructure devices</div>
<div class="section-title" style="margin-bottom:0.75rem;">Infrastructure Devices</div>
<table class="table" style="font-size:0.83rem;width:100%;">
{% for d in devices %}
<tr>
<td style="padding:0.2rem 0.5rem 0.2rem 0;">
{% set dot, label = device_state(d.state) %}{{ status_dot(dot, label) }}
<span style="color:{% if d.state == 1 %}var(--green){% else %}var(--red){% endif %};">&#9679;</span>
<span style="margin-left:0.3rem;">{{ d.name or d.mac }}</span>
</td>
<td style="padding:0.2rem 0.3rem;color:var(--text-muted);font-size:0.78rem;">{{ d.model or d.device_type or "" }}</td>
@@ -59,13 +96,7 @@
{% if d.state == 1 and d.uptime_seconds %}
<tr>
<td colspan="3" style="padding:0 0 0.3rem 1.2rem;color:var(--text-dim);font-size:0.75rem;">
up {{ d.uptime_seconds|duration }}
</td>
</tr>
{% elif d.state != 1 %}
<tr>
<td colspan="3" style="padding:0 0 0.3rem 1.2rem;color:var(--text-dim);font-size:0.75rem;">
{{ label }}
up {{ (d.uptime_seconds // 86400) }}d {{ ((d.uptime_seconds % 86400) // 3600) }}h
</td>
</tr>
{% endif %}
@@ -77,7 +108,7 @@
{# Top talkers card #}
{% if top_clients %}
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
<div class="section-title" style="margin-bottom:0.75rem;">Top talkers</div>
<div class="section-title" style="margin-bottom:0.75rem;">Top Talkers</div>
<table class="table" style="font-size:0.82rem;width:100%;">
{% for c in top_clients %}
<tr>
@@ -90,9 +121,13 @@
<td style="text-align:right;padding:0.2rem 0;white-space:nowrap;font-size:0.78rem;">
{% set rx = c.rx_rate %}{% set tx = c.tx_rate %}
<span style="color:var(--text-muted);">&#8595;</span>
{{ rx|rate }}
{% if rx >= 1048576 %}{{ "%.1f"|format(rx/1048576) }}M
{% elif rx >= 1024 %}{{ "%.0f"|format(rx/1024) }}K
{% else %}{{ "%.0f"|format(rx) }}B{% endif %}
<span style="color:var(--text-muted);margin-left:0.3rem;">&#8593;</span>
{{ tx|rate }}
{% if tx >= 1048576 %}{{ "%.1f"|format(tx/1048576) }}M
{% elif tx >= 1024 %}{{ "%.0f"|format(tx/1024) }}K
{% else %}{{ "%.0f"|format(tx) }}B{% endif %}
</td>
</tr>
{% endfor %}
@@ -103,7 +138,7 @@
{# WLAN SSIDs card #}
{% if wlan_latest %}
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
<div class="section-title" style="margin-bottom:0.75rem;">Wireless networks</div>
<div class="section-title" style="margin-bottom:0.75rem;">Wireless Networks</div>
<table class="table" style="font-size:0.83rem;width:100%;">
<tr style="color:var(--text-muted);font-size:0.78rem;">
<th style="text-align:left;font-weight:normal;padding:0 0.5rem 0.3rem 0;">SSID</th>
@@ -114,7 +149,10 @@
<tr>
<td style="padding:0.2rem 0.5rem 0.2rem 0;">{{ w.ssid }}</td>
<td style="text-align:right;padding:0.2rem 0.3rem;">{{ w.num_sta }}</td>
<td style="text-align:right;padding:0.2rem 0;" class="{% if w.satisfaction is not none %}tone-{{ 'ok' if w.satisfaction >= 80 else 'warn' if w.satisfaction >= 60 else 'crit' }}{% endif %}">
<td style="text-align:right;padding:0.2rem 0;
{% if w.satisfaction is not none %}
{% if w.satisfaction >= 80 %}color:var(--green){% elif w.satisfaction >= 60 %}color:var(--yellow){% else %}color:var(--red){% endif %}
{% endif %};">
{% if w.satisfaction is not none %}{{ w.satisfaction }}%{% else %}—{% endif %}
</td>
</tr>
@@ -126,19 +164,19 @@
{# Speedtest card #}
{% if speedtest %}
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
<div class="section-title" style="margin-bottom:0.75rem;">Last speed test</div>
<div class="section-title" style="margin-bottom:0.75rem;">Last Speed Test</div>
<div style="display:flex;gap:1.5rem;font-size:0.88rem;font-variant-numeric:tabular-nums;">
{% if speedtest.download_mbps is not none %}
<span>
<span style="color:var(--text-muted);font-size:0.8rem;">&#8595; DL</span>
<span style="margin-left:0.3rem;">{{ "%.1f"|format(speedtest.download_mbps) }}</span>
<span style="font-weight:600;margin-left:0.3rem;">{{ "%.1f"|format(speedtest.download_mbps) }}</span>
<span style="color:var(--text-muted);font-size:0.78rem;">Mbps</span>
</span>
{% endif %}
{% if speedtest.upload_mbps is not none %}
<span>
<span style="color:var(--text-muted);font-size:0.8rem;">&#8593; UL</span>
<span style="margin-left:0.3rem;">{{ "%.1f"|format(speedtest.upload_mbps) }}</span>
<span style="font-weight:600;margin-left:0.3rem;">{{ "%.1f"|format(speedtest.upload_mbps) }}</span>
<span style="color:var(--text-muted);font-size:0.78rem;">Mbps</span>
</span>
{% endif %}
@@ -152,7 +190,7 @@
{% if speedtest.server_name %}
<div style="margin-top:0.4rem;color:var(--text-dim);font-size:0.78rem;">via {{ speedtest.server_name }}</div>
{% endif %}
<div style="margin-top:0.25rem;color:var(--text-dim);font-size:0.75rem;">{{ speedtest.run_at|ago }}</div>
<div style="margin-top:0.25rem;color:var(--text-dim);font-size:0.75rem;">{{ speedtest.run_at.strftime("%Y-%m-%d %H:%M") }} UTC</div>
</div>
{% endif %}
plugins/unifi/templates/unifi/security.html
+3 -3
View File
@@ -6,7 +6,7 @@
<div class="section-title" style="margin-bottom:0.75rem;">
Active Alarms
{% if active_alarms %}
<span class="count-badge" style="float:right;">{{ active_alarms|length }}</span>
<span style="float:right;background:var(--red);color:#fff;font-size:0.72rem;padding:0.1rem 0.45rem;border-radius:10px;font-weight:600;">{{ active_alarms|length }}</span>
{% endif %}
</div>
{% if not active_alarms %}
@@ -21,7 +21,7 @@
</div>
<div>{{ al.message }}</div>
<div style="color:var(--text-dim);font-size:0.75rem;margin-top:0.1rem;">
{{ al.alarm_time|ago }}
{{ al.alarm_time.strftime("%Y-%m-%d %H:%M") }} UTC
</div>
</td>
</tr>
@@ -45,7 +45,7 @@
<div style="color:var(--text-dim);font-size:0.8rem;">{{ al.message }}</div>
</td>
<td style="padding:0.25rem 0;text-align:right;color:var(--text-dim);font-size:0.75rem;white-space:nowrap;">
{{ al.alarm_time|ago }}
{{ al.alarm_time.strftime("%m-%d %H:%M") }}
</td>
</tr>
{% endfor %}
plugins/unifi/templates/unifi/widget.html
+13 -5
View File
@@ -37,12 +37,16 @@
<span>
<span style="color:var(--text-muted);">&#8595;</span>
{% set bps = latest_wan.rx_bytes_rate %}
{{ bps|rate }}
{% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
{% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
{% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
</span>
<span>
<span style="color:var(--text-muted);">&#8593;</span>
{% set bps = latest_wan.tx_bytes_rate %}
{{ bps|rate }}
{% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
{% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
{% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
</span>
{% endif %}
</div>
@@ -55,7 +59,7 @@
<span class="ping-name" style="font-size:0.8rem;">Clients</span>
</div>
<div style="flex:1;display:flex;gap:1rem;font-size:0.82rem;">
<span style="font-weight:500;">{{ latest_clients.total_clients }}</span>
<span style="font-weight:600;">{{ latest_clients.total_clients }}</span>
<span style="color:var(--text-muted);">{{ latest_clients.wireless_clients }}w / {{ latest_clients.wired_clients }}e</span>
</div>
</div>
@@ -72,10 +76,14 @@
{% set total_bps = c.tx_rate + c.rx_rate %}
<span style="color:var(--text-muted);">&#8595;</span>
{% set rx = c.rx_rate %}
{{ rx|rate }}
{% if rx >= 1048576 %}{{ "%.1f"|format(rx / 1048576) }}M
{% elif rx >= 1024 %}{{ "%.0f"|format(rx / 1024) }}K
{% else %}{{ "%.0f"|format(rx) }}B{% endif %}
<span style="color:var(--text-muted);margin-left:0.4rem;">&#8593;</span>
{% set tx = c.tx_rate %}
{{ tx|rate }}
{% if tx >= 1048576 %}{{ "%.1f"|format(tx / 1048576) }}M
{% elif tx >= 1024 %}{{ "%.0f"|format(tx / 1024) }}K
{% else %}{{ "%.0f"|format(tx) }}B{% endif %}
</div>
</div>
{% endfor %}

Some files were not shown because too many files have changed in this diff Show More