bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Compare commits

base: bvandeusen/FabledSteward:264d44ac81a3362c7604c684712732069f981d36
Branches Tags
bvandeusen/FabledSteward:main bvandeusen/FabledSteward:dev bvandeusen/FabledSteward:renovate/configure bvandeusen/FabledSteward:feat/host-agent
..
compare: bvandeusen/FabledSteward:main
Branches Tags
bvandeusen/FabledSteward:dev bvandeusen/FabledSteward:renovate/configure bvandeusen/FabledSteward:main bvandeusen/FabledSteward:feat/host-agent
1 Commits
264d44ac81 .. main
Author SHA1 Message Date
bvandeusen 001cbafe56 Merge pull request 'Ansible schedule form: structured playbook-variable fields + first-item dropdown defaults' (#2) from dev into main
CI / lint (push) Successful in 2s
Details
CI / unit (push) Successful in 46s
Details
CI / integration (push) Successful in 2m21s
Details
CI / publish (push) Successful in 4s
Details
2026-06-30 23:44:04 -04:00
609 changed files with 8215 additions and 55120 deletions
Expand all files Collapse all files
.forgejo/workflows/ci.yml
+8 -169
View File
@@ -14,17 +14,6 @@ on:
# pull_request intentionally absent — push on [dev, main] already fires CI for
# every dev commit and dev→main PRs. Single-operator repo, no fork PRs.
# Serialize CI runs per ref so the publish lane never shares the runner's docker
# daemon with another run. Two publishes racing on one daemon evict each other's
# freshly-built image mid-push, breaking the post-build tag/push (issue #1093).
# cancel-in-progress:false is deliberate — rule 46 requires EVERY dev/main push to
# publish its own immutable :<sha> image, so a superseded run must still run to
# completion (never cancelled) to emit its SHA. (Forgejo honors `concurrency:` —
# CI-runner's build workflows use it.)
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: false
jobs:
# Fast-fail lint lane. ruff is pre-installed in the ci-python image, so this
# runs with NO dependency install and surfaces lint bounces in seconds.
@@ -69,12 +58,7 @@ jobs:
POSTGRES_USER: steward
POSTGRES_PASSWORD: steward
POSTGRES_DB: steward
# The data directory lives in RAM (#5746): postgres images declare
# VOLUME for it, so without a tmpfs each run left an anonymous volume
# that act_runner never removes, and 2,488 of them used up the runner
# host's inodes.
options: >-
--tmpfs /var/lib/postgresql/data
--health-cmd "pg_isready -U steward"
--health-interval 10s
--health-timeout 5s
@@ -85,24 +69,14 @@ jobs:
# act_runner (swarm-runner) puts service containers on the default bridge
# with no embedded DNS, so the hostname `postgres` won't resolve — we
# discover the service container's bridge IP via the docker socket. The
# job container and its Postgres share a task-number name prefix.
# job name `integration` (no underscore) is the docker-ps name filter.
run: |
set -euxo pipefail
echo "=== container landscape (diagnostic for filter scoping) ==="
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
echo "=== end landscape ==="
# The runner's daemon is shared: another repo's integration job can be
# running beside this one, and `name=integration` matches its Postgres
# too (run 8358 took the wrong one and every test failed to connect).
# Scope to this job's own task: act_runner names every container of a
# task GITEA-ACTIONS-TASK-<n>-…, so read <n> from our own container
# (its id is in the /etc/hostname bind mount) and require one match.
SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
SELF=${SELF:-$(hostname)}
TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
test -n "$TASK"
PG=$(docker ps --filter "name=$TASK" --filter "ancestor=postgres:16-alpine" -q)
test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
PG=$(docker ps --filter "name=integration" --filter "ancestor=postgres:16-alpine" -q | head -n1)
test -n "$PG"
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
test -n "$PG_IP"
export STEWARD_DATABASE_URL="postgresql+asyncpg://steward:steward@$PG_IP:5432/steward"
@@ -137,154 +111,19 @@ jobs:
# 401s with it. REGISTRY_TOKEN is a repo Actions secret holding a
# Forgejo token scoped read:package + write:package.
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.fabledsword.com -u bvandeusen --password-stdin
- name: Build (commit-SHA always; :dev on dev, :latest on main)
- name: Build + push (commit-SHA always; :dev on dev, :latest on main)
run: |
set -eux
IMAGE=git.fabledsword.com/bvandeusen/steward
# The moving tag for this ref: dev→:dev, main→:latest (rule 46). main IS
# the production line, so :latest tracks main's tip; there is no :main.
MOVING=""
if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
MOVING="dev"
elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
MOVING="latest"
fi
# Tag BOTH targets at build time (mirrors CI-runner's build workflows):
# one `docker build -t :<sha> -t :<moving>` points both tags at the image
# atomically, so we never run a post-push `docker tag` of an image a
# concurrent run could have evicted from the shared daemon (issue #1093).
if [ -n "$MOVING" ]; then
docker build -t "$IMAGE:${{ github.sha }}" -t "$IMAGE:$MOVING" .
else
docker build -t "$IMAGE:${{ github.sha }}" .
fi
# Rule 164's check, BETWEEN build and push (rule 177): an image that
# can't boot, migrate and serve with no internet never reaches a tag.
# An --internal network lets it reach its own Postgres and nothing else;
# --network none can't work for an app that needs a database to start.
# Embedded DNS works on a user-defined network, unlike the services:
# containers above (rule 79), so the app finds Postgres by name.
- name: The image boots with no internet
env:
SMOKE: steward-smoke-${{ github.run_id }}
run: |
set -eux
IMAGE=git.fabledsword.com/bvandeusen/steward:${{ github.sha }}
docker network create --internal "$SMOKE-net"
docker run -d --name "$SMOKE-pg" --network "$SMOKE-net" \
--tmpfs /var/lib/postgresql/data \
-e POSTGRES_USER=steward -e POSTGRES_PASSWORD=steward -e POSTGRES_DB=steward \
postgres:16-alpine
# Waits with deadlines (rule 156); pg_isready, not /dev/tcp, which is
# a bash builtin and never succeeds under sh.
ready=""
for _ in $(seq 1 60); do
if docker exec "$SMOKE-pg" pg_isready -U steward -d steward -q; then ready=1; break; fi
sleep 2
done
test -n "$ready" || { echo "FATAL: postgres not ready within 120s"; exit 1; }
docker run -d --name "$SMOKE-app" --network "$SMOKE-net" \
-e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/steward" \
-e STEWARD_SECRET_KEY=smoke-test-placeholder \
"$IMAGE"
# The image has no curl; its own Python fetches. Prints the body.
fetch() {
docker exec "$SMOKE-app" python -c 'import sys, urllib.request
r = urllib.request.urlopen("http://127.0.0.1:5000" + sys.argv[1], timeout=5)
sys.stdout.write(r.read().decode())' "$1"
}
# Migrations run before the server starts, so /health answering
# means the schema applied against a real database.
up=""
for _ in $(seq 1 60); do
if fetch /health >/dev/null 2>&1; then up=1; break; fi
sleep 2
done
if [ -z "$up" ]; then
echo "FATAL: Steward never answered /health with no internet access"
docker logs "$SMOKE-app" 2>&1 | tail -80
exit 1
fi
fetch /health | grep -q '"ok"'
# tini is PID 1, so zombies are reaped and SIGTERM reaches Steward.
test "$(docker exec "$SMOKE-app" cat /proc/1/comm)" = "tini"
# The zone database is in the image, so a reader's zone resolves
# instead of quietly falling back to UTC (milestone 490).
docker exec "$SMOKE-app" python -c 'from zoneinfo import ZoneInfo; ZoneInfo("America/New_York")'
# A fresh database lands on first-run setup: a real page, its assets
# served by Steward itself, and no third-party host in it (rule 155).
fetch / > /tmp/smoke-page.html
grep -qi "<!doctype html" /tmp/smoke-page.html
grep -q "/static/" /tmp/smoke-page.html
if grep -Eqi 'https?://(fonts\.|cdn\.|unpkg\.|cdnjs\.|ajax\.)' /tmp/smoke-page.html; then
echo "FATAL: the page references a third-party host"
grep -Eoi 'https?://[^"'"'"' ]+' /tmp/smoke-page.html
exit 1
fi
# A backup made by the image restores into an empty database, and
# Steward starts on it (milestone 507). It carries configuration
# (the marker setting) and leaves metrics' rows out.
sql() { docker exec "$SMOKE-pg" psql -U steward -v ON_ERROR_STOP=1 -tA "$@"; }
sql -d steward -c "INSERT INTO app_settings (key, value_json, updated_at) VALUES ('smoke.backup', '\"kept\"', now())"
sql -d steward -c "INSERT INTO plugin_metrics (id, source_module, resource_name, metric_name, value, recorded_at) VALUES ('smoke-metric', 'smoke', 'smoke', 'smoke', 1, now())"
docker exec -u app "$SMOKE-app" steward backup --to /tmp/backups
sql -d steward -c "CREATE DATABASE restored"
docker exec -u app \
-e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/restored" \
"$SMOKE-app" sh -c 'steward restore /tmp/backups/steward-*.tar'
test "$(sql -d restored -c "SELECT value_json FROM app_settings WHERE key = 'smoke.backup'")" = '"kept"'
test "$(sql -d restored -c "SELECT count(*) FROM plugin_metrics")" = "0"
test "$(sql -d restored -c "SELECT count(*) FROM alembic_version")" -ge 1
docker run -d --name "$SMOKE-restored" --network "$SMOKE-net" \
-e "STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@$SMOKE-pg:5432/restored" \
-e STEWARD_SECRET_KEY=smoke-test-placeholder \
"$IMAGE"
up=""
for _ in $(seq 1 60); do
if docker exec "$SMOKE-restored" python -c 'import urllib.request
urllib.request.urlopen("http://127.0.0.1:5000/health", timeout=5)' >/dev/null 2>&1; then up=1; break; fi
sleep 2
done
if [ -z "$up" ]; then
echo "FATAL: Steward never started on the restored database"
docker logs "$SMOKE-restored" 2>&1 | tail -80
exit 1
fi
# A stop is a clean shutdown: SIGTERM through tini to Hypercorn, which
# finishes within its graceful timeout and exits 0.
docker stop -t 30 "$SMOKE-app"
code="$(docker inspect -f '{{.State.ExitCode}}' "$SMOKE-app")"
test "$code" = "0" || { echo "FATAL: Steward exited $code on SIGTERM"; exit 1; }
- name: Push
run: |
set -eux
set -euxo pipefail
IMAGE=git.fabledsword.com/bvandeusen/steward
docker build -t "$IMAGE:${{ github.sha }}" .
docker push "$IMAGE:${{ github.sha }}"
if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:dev"
docker push "$IMAGE:dev"
elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:latest"
docker push "$IMAGE:latest"
fi
- name: Smoke logs
if: always()
run: docker logs "steward-smoke-${{ github.run_id }}-app" 2>&1 | tail -60 || true
# A failed smoke must not leave a Postgres and an app on the shared daemon.
- name: Smoke clean-up
if: always()
run: |
docker rm -fv "steward-smoke-${{ github.run_id }}-app" "steward-smoke-${{ github.run_id }}-restored" \
"steward-smoke-${{ github.run_id }}-pg" >/dev/null 2>&1 || true
docker network rm "steward-smoke-${{ github.run_id }}-net" >/dev/null 2>&1 || true
- name: Prune dangling layers
if: always()
run: docker image prune -f
Dockerfile
+7 -20
View File
@@ -1,5 +1,4 @@
# The Python CI tests on (ci-python:3.14), so the image runs what was tested.
FROM python:3.14-slim
FROM python:3.13-slim
RUN DEBIAN_FRONTEND=noninteractive apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
@@ -11,17 +10,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \
# sshpass — Ansible shells out to it for first-contact password SSH
# (provisioning a fresh host before the managed key is installed)
sshpass \
# git — clones and pulls Ansible git sources (Settings > Ansible)
git \
# tini — PID 1: reaps the zombies ssh and ansible leave behind and
# forwards SIGTERM to Steward so a stop is a graceful shutdown
tini \
# tzdata — the zone database, so times show in each reader's own zone
# (milestone 490); Debian's slim base no longer carries it
tzdata \
# postgresql-client-17 — pg_dump and pg_restore for Steward's own
# backups (milestone 507); 17 reads and writes a Postgres 16 server
postgresql-client-17 \
&& rm -rf /var/lib/apt/lists/*
# Upgrade pip to suppress the version notice
@@ -31,11 +19,10 @@ WORKDIR /app
COPY pyproject.toml .
COPY steward/ steward/
# Bundle the extras whose features ship in the image: [ansible] for the
# playbook runner, [snmp] (pysnmp) for the SNMP poller, [ldap] (ldap3, pure
# Python) for LDAP sign-in under Settings -> Auth. Without them those features
# load but fail for want of their runtime dependency.
RUN pip install --no-cache-dir '.[ansible,snmp,ldap]'
# Bundle the extras whose first-party plugins ship in the image: [ansible] for
# the playbook runner, [snmp] (pysnmp) for the SNMP poller. Without them those
# bundled plugins load but silently no-op for want of their runtime dependency.
RUN pip install --no-cache-dir '.[ansible,snmp]'
COPY alembic.ini .
# First-party plugins ship inside the image (bundled root at /app/plugins).
@@ -49,10 +36,10 @@ RUN chmod +x /entrypoint.sh
# so the entrypoint can fix up /data permissions if needed, then drops to
# 'app' (uid 1000) via gosu before launching steward.
RUN useradd -m -u 1000 app \
&& mkdir -p /data/ansible/sources /data/ansible/runs /data/ansible/playbooks /data/plugins \
&& mkdir -p /data/playbook_cache /data/plugins \
&& chown -R app:app /data /app
EXPOSE 5000
ENTRYPOINT ["/usr/bin/tini", "--", "/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["steward", "--host", "0.0.0.0", "--port", "5000"]
README.md
-3
View File
@@ -119,10 +119,7 @@ See `docs/plugins/` for a full plugin development guide.
| `docs/core/monitors.md` | Ping and DNS monitors: probe logic, metrics emitted, data models |
| `docs/core/alerting.md` | Alert rules, state machine, notification channels, template variables |
| `docs/core/ansible.md` | Playbook sources, run lifecycle, SSE streaming |
| `docs/core/backups.md` | Steward's daily backup to an S3 bucket (Fabled Archives), what it holds, and `steward restore` |
| `docs/plugins/overview.md` | Plugin system: how loading works, plugin.yaml schema, required exports |
| `docs/plugins/writing-a-plugin.md` | Step-by-step plugin development guide |
| `docs/plugins/traefik.md` | Traefik plugin: config, metrics, alert rule examples |
| `docs/plugins/prometheus.md` | Prometheus plugin: scraping any /metrics endpoint, how series are stored, the Fabled Archives recipe |
| `docs/plugins/prometheus-widgets.md` | Prometheus widget templates: writing, sharing and copying one in, every block, the format reference |
| `docs/reference/code-map.md` | Where every key function, model, and route lives in the codebase |
ci-requirements.md
+1 -1
View File
@@ -46,7 +46,7 @@ git.fabledsword.com/bvandeusen/ci-python:3.14
- Steward has **no frontend and no Redis/Celery** (no external workers), so there
is no frontend-build lane and the only service container is Postgres.
- Integration uses Fabled-Git Actions `services:` + a socket-discovered bridge IP
- Integration uses Forgejo Actions `services:` + a socket-discovered bridge IP
because `act_runner` (swarm-runner v0.6+) puts services on the default bridge
with no embedded DNS. FabledCurator's `ci.yml` is the canonical example of the
pattern; Steward's is the same shape minus Redis and minus sharding.
compose.deploy.yml
+7 -4
View File
@@ -39,7 +39,8 @@ services:
# mount, pin STEWARD_SECRET_KEY (env or a docker secret) instead, so the key
# never depends on filesystem ownership being correct on every node.
depends_on:
- db
db:
condition: service_healthy
restart: unless-stopped
db:
@@ -53,9 +54,11 @@ services:
volumes:
- pgdata:/var/lib/postgresql/data
# No host port published — db is reachable only on the backend network.
# No healthcheck: a slow disk (NFS) failed pg_isready and the orchestrator
# killed a healthy database mid-write (#5275). Steward waits for the
# database itself, at startup and per request (steward/database.py).
healthcheck:
test: ["CMD-SHELL", "pg_isready -U steward"]
interval: 5s
timeout: 5s
retries: 5
restart: unless-stopped
networks:
config.example.yaml
-8
View File
@@ -13,14 +13,6 @@
database:
url: "postgresql+asyncpg://steward:password@localhost/steward"
# Optional: seconds to wait at startup for the database to become reachable
# before giving up (default: 60). Steward retries with backoff rather than
# crashing when the DB is merely slow to arrive — replaying WAL after an
# unclean shutdown, or container DNS not yet resolving after a host reboot.
# Raise it if your database is routinely slower than this to accept
# connections. Env var: STEWARD_DB_CONNECT_TIMEOUT
# connect_timeout: 60
# Optional: override the auto-generated secret key.
# If not set, a key is auto-generated on first run and saved to /data/secret.key.
# secret_key: "change-me-to-a-random-string"
docker-compose.yml
+7 -4
View File
@@ -27,7 +27,8 @@ services:
- STEWARD_SECRET_KEY=dev-secret-key-change-me
- PYTHONPATH=/app
depends_on:
- db
db:
condition: service_healthy
restart: unless-stopped
db:
@@ -41,9 +42,11 @@ services:
- "5432:5432"
volumes:
- pgdata:/var/lib/postgresql/data
# No healthcheck: a slow disk (NFS) failed pg_isready and the orchestrator
# killed a healthy database mid-write (#5275). Steward waits for the
# database itself, at startup and per request (steward/database.py).
healthcheck:
test: ["CMD-SHELL", "pg_isready -U steward"]
interval: 5s
timeout: 5s
retries: 5
restart: unless-stopped
volumes:
docs/architecture.md
-66
View File
@@ -44,30 +44,6 @@ All routes are Quart Blueprints registered in `app.py`:
Plugin blueprints are mounted automatically by `load_plugins()` using the plugin directory name as the URL prefix.
`/ai-access/` (`ai_access_bp`, `steward/ai_access/routes.py`) is where a signed-in user manages their API tokens.
### `/mcp`: outside the blueprint pipeline
`mount_mcp(app)` (`steward/mcp/server.py`) wraps `app.asgi_app`. A request to
`/mcp` is handed to the MCP SDK's streamable-HTTP app before Quart sees it, so
no blueprint, session cookie or `before_request` hook runs for it. Each
request then goes through these steps:
1. **Database gate.** If the database is unreachable, the request gets a 503
with a readable message.
2. **Bearer resolution.** `api_tokens.resolve_token()` turns the token into a
`Caller` (user, effective role, token id). A missing, revoked or expired
token gets a 401.
3. **Caller context.** The `Caller` and the app are set in ContextVars
(`steward/mcp/_context.py`). Tools read them through `current_caller()` and
`mcp_app()`, because Quart's `current_app` and `session` do not exist there.
4. **Role gate.** `StewardMCPServer.refusal()` hides and refuses tools above
the caller's role, and plugin tools whose plugin is disabled.
The SDK's session manager runs in Quart's `before_serving`/`after_serving`
lifecycle. The server is stateless: no MCP session outlives a request. See
[MCP server](core/mcp.md) for the security model and the tools.
---
## Scheduler
@@ -87,48 +63,6 @@ Core tasks registered in `app._task_registry`:
---
## Metric Store
Every plugin and monitor records into one store keyed by (source, resource, metric):
- `plugin_metrics` holds raw samples for `metrics.retention.raw_days` (7 by default).
- `plugin_metric_latest` holds each key's newest value.
- `data_cleanup` rolls raw samples older than the raw window into `plugin_metrics_hourly`, kept for `metrics.retention.rollup_days` (90 by default).
Each rolled-up hour keeps three figures:
- `value_avg`: the hour's average.
- `value_max`: its peak.
- `value_p95`: the level it held for all but its briefest 5%, about 3 minutes.
A two-minute spike raises the peak and leaves the p95 low; an hour spent at that level raises both. Learned normals use this to tell a recurring spike from a held level. Hours rolled up before the p95 was kept have it NULL; read those as their average.
### Normals and anomalies
A metric a plugin registers with `learn_normal()` gets a normal per hour of the day, then per hour of the week once three weeks of history exist (`metric_norms`, relearned hourly by `metric_norms`; `steward/core/norms.py`).
Every minute, `metric_anomalies` averages each learned key's last 15 minutes and compares it with the normal for that hour. A level above the usual hold ×1.25, and above the metric's floor, opens an anomaly (`steward/core/anomalies.py`):
- `metric_anomalies` holds one row per item to review. A repeat while the row still waits joins it as another occurrence (`metric_anomaly_spans`).
- A row is `to_review` until someone marks it `normal` or `real` (Alerts → Anomalies, or MCP `review_anomaly`). An ended row nobody reviews within `anomalies.review_days` (14) becomes `lapsed`.
- Learning skips every hour an occurrence covers unless its anomaly was marked `normal`, so a new level is never learned without review.
- `data_cleanup` prunes reviewed and lapsed rows at `data.retention_days`; a row still to review is never pruned.
What learns a normal, and the floor each departure must clear (Settings → Thresholds → Normals, `norms.floor.*`; a floor may be a function, `floor_setting()`, read at each pass):
| Kind | Metric | Floor (default) |
|---|---|---|
| Host CPU | `host_agent` / `cpu_pct` | 10 points |
| Host memory | `host_agent` / `mem_used_pct` | 5 points |
| Host load per core | `host_agent` / `load_pct` (load_1m ÷ cores × 100, recorded at ingest) | 10 points |
| Container CPU | `docker` / `cpu_pct` (100% is one core) | 10 points of one core |
| Container memory | `docker` / `mem_pct` (of its limit, or of host RAM when it has none) | 5 points |
| Monitor response time | `<type>` / `response_ms` | 100 ms |
| Router 4xx share, p95 | `traefik` / `error_rate_4xx_pct`, `latency_p95_ms` | 2 points, 50 ms |
Host metrics learn with `parts=False`: the host learns, its cores (`<host>:core<N>`) don't. Disk space, a monitor that is down, an agent gone quiet and a router's 5xx share are failures and keep their fixed cutoffs.
Pages grade by the normal through `anomalies.graded()` (hosts' `host_grades` and `worst_reading(normals=)`, `docker.service.usage(thresholds=, norms=)`, `monitors.service.latency_grade`): a reading past its cutoff but within its normal reads healthy and says what is usual at this hour. The host, monitor, container and Traefik pages list their anomalies with `_anomaly_card.html`.
---
## Database Session Pattern
`app.db_sessionmaker` is an SQLAlchemy `async_sessionmaker`. All DB access follows this pattern:
docs/core/alerting.md
+12 -130
View File
@@ -8,7 +8,7 @@ Alert rules evaluate every metric that flows through `record_metric()`. There is
When any monitor or plugin calls `record_metric()`:
1. The metric value is written to the `plugin_metrics` table, and `plugin_metric_latest` moves to it if it is the newest sample of that metric
1. The metric value is written to the `plugin_metrics` table
2. All enabled `AlertRule` rows matching `(source_module, resource_name, metric_name)` are loaded
3. Each matching rule is evaluated against the new value
4. If a state transition occurs, an `AlertEvent` row is written
@@ -32,25 +32,18 @@ Notifications are sent via `asyncio.create_task()` after the function returns, s
Each `AlertRule` has one associated `AlertState` row. The state transitions are:
```
inactive ──(breached)──► pending ──(N breaching checks in a row)──► FIRING
▲ │ │
└──(an OK check)───────┘ │
FIRING ──(acknowledged)──► ACKNOWLEDGED
FIRING / ACKNOWLEDGED ──(M OK checks in a row)──► RESOLVED ──► inactive
inactive ──(breached)──► pending ──(consecutive count met)──► FIRING
│ │
└──(recovered)──► inactive (no notification) │
│
FIRING ──(recovered)──► RESOLVED ──► inactive
FIRING ──(acknowledged)──► ACKNOWLEDGED
ACKNOWLEDGED ──(recovered)──► RESOLVED ──► inactive
ACKNOWLEDGED ──(re-breached)──► FIRING
```
N is the rule's consecutive checks, M its checks to clear. An acknowledgement
holds for as long as the breach lasts: only a resolve clears it, and a breaching
check never sends an acknowledged alert back to firing.
`RESOLVED` is transient — the evaluator writes the event and immediately sets state back to `inactive` within the same transaction. `RESOLVED` never persists as a final state in the DB.
Every transition is an `AlertEvent`, the history the Alerts → History page and
MCP `get_alert_history` read. A transition a person made carries them in
`actor_id`: an acknowledgement (firing → acknowledged), or a rule edit, disable
or enable that reset the alert (→ resolved, with a note such as "rule
disabled"). Transitions caused by readings, or by Steward itself, have none.
### State Definitions
| State | Meaning |
@@ -58,7 +51,7 @@ disabled"). Transitions caused by readings, or by Steward itself, have none.
| `inactive` | Threshold not breached |
| `pending` | Threshold breached but consecutive count not yet met; no notification sent |
| `firing` | Consecutive count met; FIRING notification sent |
| `acknowledged` | Someone has seen it; no further notifications; clears when the readings recover |
| `acknowledged` | Operator acknowledged; suppresses repeat notifications; auto-clears on recovery |
| `resolved` | Transient — notification sent, immediately transitions to `inactive` |
---
@@ -75,39 +68,6 @@ Alert rules are created in the UI at `/alerts/`. Each rule requires:
- **Threshold** — numeric value
- **Consecutive failures required** — how many consecutive breaches before FIRING (default 1)
The form offers only what has reported, read from `plugin_metric_latest`.
For the host agent, a **Host** is picked first, then a **Part** of it: "The
whole host", then its mounts, CPU cores, disks and so on, under a heading for
each kind. A part that reports nothing a rule can watch, such as an interface's
traffic, isn't listed. The **Metric** list holds only what the picked resource
reports. Under it, a preview shows the metric's current reading and its last
24 hours (lowest to highest, with an hourly sparkline; it loads just after the
page, since a day of samples can take a while to read), so the threshold can be
set against real numbers.
### Rules that cover hosts
A rule on `host_agent` can cover **every host**, or **every host in an
inventory group**, instead of naming one resource (#5464). Inventory groups
are Steward's grouping system: a host is in a group through its linked
Ansible target.
A covering rule has no state of its own. Steward keeps one ordinary rule per
covered host under it, named `<rule>: <host>`, so each host's alert keeps its
own state, history and acknowledgement. Each minute (the `alert_reconcile`
task), and whenever the rule is saved:
- a host that has reported in the last 7 days, and is covered, gets a rule;
- a host that is no longer covered loses its rule, and that rule's history;
- the hosts' rules take the covering rule's condition, checks, notification
setting, action and on/off.
A host's rule is changed through its covering rule; editing, disabling or
deleting it alone is refused. What a rule covers is fixed once it exists, but
a group rule can move to another group. Over MCP, `create_alert_rule` takes
`covers` ("all_hosts" or "group") and `group`, and `list_alert_rules` shows
each covering rule's `hosts` and each host's rule's `part_of`.
### Available Metrics by Module
| `source_module` | `metric_name` | Description |
@@ -123,24 +83,6 @@ each covering rule's `hosts` and each host's rule's `part_of`.
| `traefik` | `latency_p95_ms` | Approximate p95 latency (ms) |
| `traefik` | `latency_p99_ms` | Approximate p99 latency (ms) |
#### Steward itself
Source `steward`, resource `process`, recorded once a minute by the
self-monitor tick (`steward/core/self_monitor.py`; the ingest tally is in
`steward/core/health.py`). Settings → System (`/settings/system/`, admin)
shows each one now and over the last day, with a verdict above that names
failed agent posts, a gap in watching, or open files near the limit.
| `metric_name` | Description |
|---|---|
| `ingest_posts` | Host-agent posts Steward took in the last minute |
| `ingest_failed` | Posts in the last minute that didn't land (handler raised, or the agent gave up) |
| `ingest_slowest_ms` | The slowest post in the last minute; agents give up at 10 s |
| `db_roundtrip_ms` | A `SELECT 1`, connection checkout included |
| `unwatched_secs` | The gap before this tick when it exceeds `watch.GAP_S` (a deploy or an outage); 0 when unbroken |
| `cleanup_secs` | How long the last hourly retention and rollup run took |
| `open_fds` / `open_fds_pct` | Steward's open files, and as a share of its limit (absent without `/proc`) |
---
## Notifications
@@ -162,43 +104,6 @@ Configured at `/settings/` under SMTP. Settings:
Email is skipped if `smtp.host` is empty.
Alerts that arrive together go out as one email. The first alert opens a
2-minute window. When it closes, each recipient gets one message listing every
alert in the window, oldest first, in their own zone, with a subject like
"[Steward] 3 alerts: 2 firing, 1 resolved". One alert on its own keeps its
usual email.
Every email Steward sends (alerts, the weekly report, invites, password
resets and the test buttons) waits in the database (`email_outbox`,
`core/outbox.py`) until the mail server accepts it, so nothing waiting is lost
when Steward restarts: the sender picks it up at boot. The sender runs every 15
seconds and settles each email on the server's answer:
- **Accepted:** sent.
- **Refused** (an SMTP 5xx, such as "no such mailbox"): failed at once, with
the server's words. Trying again would only repeat the refusal.
- **Temporary** (no connection, a timeout, a 4xx): tried again after 30 s,
1, 2, 5, 10 and then every 15 minutes. An email the server still hasn't
taken an hour after its first try is failed with the last error.
Once every email carrying an alert has settled, that alert's event records
whether the email went, and to whom it didn't.
Where to see it: each alert's row in **Alerts → History** says what its email
is doing (gathering, with when it sends; retrying, with the attempt, the next
try and the server's last answer; sent; failed, with who wasn't reached and
why), and `get_alert_history` over MCP reports the same. **Settings → System →
Outgoing email** lists every email of the last 7 days not yet sent, failures
first, and the page's verdict names failures and any email still retrying
after 5 minutes.
An email a person is waiting on is tried at once instead of on the next tick:
the weekly report's **Send now**, an invite, and the test buttons, so the page
can say what the server answered. A report or invite the server couldn't take
yet stays queued and is retried as above. A test (the Settings → Notifications
test email, a rule's **Send test**) is recorded but never retried: its one
answer is the result.
### Webhook
Configured at `/settings/` under Webhook. The body is a Jinja2 template rendered to JSON. Content-Type is always `application/json`. If the rendered template is not valid JSON, the delivery is logged as failed and no request is sent.
@@ -219,29 +124,7 @@ Default template (Discord-compatible):
{"content": "**{{ alert.state }}** — {{ alert.resource }} — {{ alert.rule_name }} ({{ alert.metric }} = {{ alert.value }})"}
```
Webhook is skipped if `webhook.url` is empty. Webhooks are not gathered into
bursts: each alert posts on its own, at once.
### Send test
**Send test** at the top of a rule's page (operator role), or `send_test_alert` over MCP,
sends a `TEST` alert for that rule now, by email and webhook, the same way a
real alert would go. It uses the rule's name and condition and the latest
reading of what the rule watches. A rule that covers hosts uses its first
host. Each channel's result shows under the button. Nothing fires, the rule's
history doesn't change, and the test is recorded in the audit log as
`alert_rule.tested`.
### On the dashboard
The **Alerts** widget (`/alerts/widget`) sits beside the Overview in the
recommended layout. It opens with a verdict ("2 firing, 1 silent." or
"Nothing firing: all 12 rules are quiet."). Below that it lists what is
firing or silent: each rule, what it watches, its reading now against what
it fires on, and since when. Acknowledged alerts follow, with who
acknowledged them and when. A line appears when alert email has failed in
the last 7 days or is still retrying, linking admins to Settings → System.
The quiet rules are counted at the end.
Webhook is skipped if `webhook.url` is empty.
---
@@ -252,5 +135,4 @@ Defined in `steward/models/alerts.py`:
- **`alert_rules`** — one row per configured rule
- **`alert_states`** — one row per rule, tracks current state and consecutive failure count
- **`alert_events`** — append-only log of all state transitions and notification outcomes
- **`plugin_metrics`** — all metric values written by any monitor or plugin. Its series index carries the value (`ix_plugin_metrics_series_value`), so a day of one metric is read from the index alone even when nothing is cached; Steward builds it in the background after startup (`core/metric_index.py`), never during a migration
- **`plugin_metric_latest`** — the newest sample of each (source, resource, metric). Pages that show "what it is now" read this instead of searching history; retention drops a row once its sample leaves the raw window
- **`plugin_metrics`** — all metric values written by any monitor or plugin
docs/core/ansible.md
+3 -10
View File
@@ -31,9 +31,10 @@ ansible.sources:
url: "https://github.com/example/infra.git"
branch: "main"
pull_interval_seconds: 300
cache_path: "/data/playbook_cache/infra-repo"
```
Each git source is checked out under `/data/ansible/sources/<name>` (override with `STEWARD_ANSIBLE_SOURCE_DIR`), on the persistent volume, so a redeploy keeps it. One scheduled task, `ansible_git_pulls`, ticks every 30 s and pulls each source once its `pull_interval_seconds` has passed; a source added in Settings is pulled within a tick, with no restart. A clone or pull that runs past 120 s is cut (its process group terminated), a clone that fails is removed so the next attempt starts clean, and git never prompts for credentials.
Git sources register a `ScheduledTask` named `ansible_git_pull_<name>` that runs `git pull` on the configured interval.
---
@@ -101,20 +102,12 @@ If a client connects after the run has already completed, the endpoint immediate
| Status | Description |
|---|---|
| `queued` | Waiting for a host another run holds, or for a run slot |
| `running` | Execution in progress |
| `success` | Playbook exited 0 |
| `failed` | Playbook exited non-zero |
| `interrupted` | App restarted while run was in progress |
| `cancelled` | Stopped by an operator, or by Steward shutting down mid-run |
On startup, the app marks any runs still `running` or `queued` as `interrupted` (see `_mark_interrupted_runs()` in `app.py`).
### Overlapping runs
- **One run per host.** Before it takes a run slot, a run claims every host it touches. Hosts come from the inventory Steward writes, keyed by address. A run on a source's own inventory file claims that file, because Steward doesn't read its hosts. While another run holds one of its hosts, the run shows `queued` and says which run it waits for. After an hour of waiting it fails and says so.
- **One run at a time per schedule.** A schedule that falls due while its previous run is still `queued` or `running` skips that firing. The Schedules page shows "Skipped: the previous run was still going" beside that run. The next firing comes an interval later.
- **One action run at a time per alert rule.** When a rule fires again while the action run from an earlier firing is still going, no new run starts. The firing's event says it was skipped and names the run.
On startup, the app marks any runs still in `running` state as `interrupted` (see `_mark_interrupted_runs()` in `app.py`).
Output stored in the DB is capped at 1 MB. If truncated, `[output truncated]` is appended to the DB column, but the live SSE stream continues unaffected.
docs/core/backups.md
-117
View File
@@ -1,117 +0,0 @@
# Backups: Steward backing itself up, and coming back
Steward sends a backup of itself to an S3 bucket once a day. Fabled Archives
is the intended target. A backup is one file, and `steward restore` brings
Steward back from it into an empty database.
## What a backup holds
`steward-<UTC time>.tar`, containing:
| Member | What it is |
|---|---|
| `steward.dump` | `pg_dump --format=custom` of Steward's database |
| `secret.key` | The key Steward's stored passwords and tokens are sealed with |
| `manifest.json` | Format, Steward version, migration heads, time, and the tables whose rows were left out |
**Kept:**
- configuration: settings, users, API tokens, hosts, monitors, alert rules, maintenance windows, Ansible inventory and schedules, dashboards, SNMP devices, Prometheus targets, UniFi client names;
- the audit log;
- alert history.
**Left out** (the tables come back empty and fill again as Steward runs):
- metrics and their rollups;
- monitor check results;
- Docker, Traefik and UniFi history;
- Ansible run records;
- the email queue;
- password-reset links.
The list is `steward/backup/contents.py`. A test fails when a new table
appears in neither list, so whoever adds a table decides.
**The secret key is in the file.** That makes a restore complete on its own.
It also means anyone holding a copy can read every password and key Steward
keeps. Keep the bucket private.
## Setting it up with Fabled Archives
1. In Archives, make a backup target for Steward with an S3 key. The default
append-only key is enough: Steward only uploads and never deletes.
How long backups are kept is set on the Archives side.
2. In Steward, open **Settings → Backups** and fill in:
- **Endpoint**: the S3 address Archives shows for the target, without the
bucket (for example `http://archives:8081`). Requests are path-style:
`endpoint/bucket/file`.
- **Bucket**: the target's bucket name.
- **Access key** and **Secret key**: from Archives. The secret is sealed at
rest; a blank box on save keeps the stored one.
3. Press **Test the bucket**. A key that may not list ("AccessDenied") still
counts as reached: an append-only key can't list, and uploads work anyway.
4. Tick **Back up every day**, pick the hour (on your clock), and save.
**Back up now** makes one straight away.
A day missed while Steward was down is made up when it comes back. A failed
backup is tried again an hour later, never sooner. The last 30 runs are
listed on the page, and each one is in the audit log (`backup.made` /
`backup.failed`).
### Alert when backups stop
While backups are on, Steward records two metrics under the source
**Steward itself**, resource `backup`:
| Metric | Meaning |
|---|---|
| `backup_age_secs` ("Newest backup's age") | Seconds since the last backup was sent, updated every 15 minutes |
| `backup_bytes` ("Backup size") | The size of the last backup sent |
A rule such as `backup_age_secs > 93600` (26 hours) fires when a daily backup
has been missed.
## Making a backup by hand
```sh
docker exec -u app <steward container> steward backup --to /tmp/backups
```
This writes the file and sends it nowhere. It needs the same database access
as Steward itself.
## Restoring
`steward restore` writes into an **empty** database. Pass `--wipe` to drop
everything in the database's public schema first.
1. Stop Steward (scale its service to 0) so nothing writes during the restore.
2. Download the backup file from Archives.
3. Run the restore with the image, pointed at the database:
```sh
docker run --rm \
-e STEWARD_DATABASE_URL=postgresql+asyncpg://steward:…@db:5432/steward \
-v steward_data:/data -v "$PWD":/restore \
git.fabledsword.com/bvandeusen/steward:latest \
steward restore /restore/steward-20261012T030000Z.tar
```
- The backup's secret key is written to `/data/secret.key`.
- If `STEWARD_SECRET_KEY` is set instead, it must equal the backup's key,
or the restore stops and says so. Read the key with
`tar -xOf <file> secret.key`.
4. Start Steward. On start it runs any migrations the backup is behind on.
**How the restore works.** pg_restore runs in its three sections: schema,
then data, then constraints and indexes. Between the data and the
constraints, a kept row that points at a left-out row is put right by its
foreign key's ON DELETE:
- an alert event's link to an Ansible run is cleared;
- a row whose foreign key would cascade or block is removed.
pg_restore 17 writes each section as a script for psql. It runs against
Postgres 16 with one setting line dropped (`transaction_timeout`, which 16
doesn't know).
CI checks the whole path on every build. The built image backs up its own
database, restores into a second one, checks a setting came back and the
metrics table is empty, then starts Steward on it.
docs/core/mcp.md
-134
View File
@@ -1,134 +0,0 @@
# MCP server
Steward serves the [Model Context Protocol](https://modelcontextprotocol.io) at
`/mcp`. An AI client connected to it can look into everything Steward watches
(hosts, monitors, alerts, metrics, containers, playbook runs) and, with a
high enough role, change it. A token's role sets that ceiling.
**Source:** `steward/mcp/`. The tools live in `steward/mcp/tools/`, and a
plugin's tools live in `plugins/<name>/mcp_tools.py`.
**Transport:** streamable HTTP, stateless. Every request carries its own
token, so a client survives a Steward restart without reconnecting.
---
## Connecting
1. In Steward, open **AI access** and create a token. Pick the lowest role
that does the job.
2. Copy the token (it is shown once) and use one of the recipes on that
page. They carry the real endpoint URL and the new token. For Claude Code:
```sh
claude mcp add --transport http steward https://steward.example/mcp \
--header "Authorization: Bearer stw_…"
```
3. Ask the client to call `whoami`. It answers with the user, the effective
role and what that role allows.
Claude Desktop connects through `mcp-remote`. Open WebUI and other clients
need an MCP server with the **Streamable HTTP** transport and bearer
authentication. The AI access page shows the recipes for both.
### Behind a reverse proxy
- The URL on the AI access page is **Settings → General → Public base URL**
plus `/mcp`. If that setting is empty, the URL is built from the request's
host instead.
- The proxy must pass the `Authorization` header through unchanged.
- It must not buffer responses. Replies may arrive as server-sent events
(`text/event-stream`). For nginx, set `proxy_buffering off;` on the
location.
- Steward does no Host-header (DNS-rebinding) check on `/mcp`, because the
bearer token is the security boundary. Serve it over HTTPS.
---
## Security model
| Rule | How it is enforced |
|---|---|
| Bearer tokens only, no OAuth | `steward/core/api_tokens.py`. Only a hash is stored, and the token is shown once. |
| A token never acts above its user | Effective role = min(token role, user's **current** role). Lowering a user lowers every token they hold; disabling the user stops them all. |
| Default deny | A tool is reachable only if it was registered with a role (`register_tool(fn, role=..., read_only=...)`). `tools/list` shows a caller only the tools their role allows. A call above the role is refused, and the refusal names the role it needs. |
| Plugin tools follow the plugin | A disabled plugin's tools are hidden and refused. |
| Dry run first | `run_playbook`, `create_schedule` and `set_alert_action` default to Ansible check mode (`--check --diff`). Changes are applied only when the caller passes `check=false`. A playbook its author flagged `# steward:confirm: true` also needs `confirmed=true`. `prune_docker` previews unless `check=false`. |
| Secrets stay put | Playbook variables Steward detects as secret reach the run but are never stored, audited or returned. Schedules and alert actions refuse them, since they would have to store them. Inventory variables are listed by name only, and HTTP monitor header values are redacted. |
| Every change is audited | Each write records an audit event as `<user> via MCP`, with the token id in its detail. `list_audit_events(user="<user>")` includes those events. |
| Unknown arguments are refused | A misspelt argument is rejected with a "did you mean", rather than silently dropped. |
Tokens never mint or revoke tokens; that happens only on the AI access
page, in a browser session.
---
## Tools by role
Each role includes the tools of the roles above it in this list.
### Viewer: read everything
| Area | Tools |
|---|---|
| Start here | `whoami`, `get_overview` (what is wrong now, ranked, each finding with the call that shows its evidence) |
| Hosts and metrics | `list_hosts`, `get_host`, `get_host_metrics`, `query_metrics` |
| Monitors | `list_monitors`, `get_monitor` |
| Alerts | `list_alerts`, `get_alert_history`, `list_alert_rules`, `get_alert_rule_options`, `list_maintenance_windows` |
| Metric anomalies | `list_anomalies` (metrics that left their normal for the hour, to review first) |
| Automation | `list_playbooks`, `get_playbook`, `list_inventory`, `list_ansible_runs`, `get_ansible_run`, `wait_for_run`, `list_schedules` |
| Audit | `list_audit_events` |
| Docker plugin | `list_containers`, `get_container`, `list_swarm_services`, `search_container_logs`, `get_docker_disk_usage` |
| Host agent plugin | `list_agents` |
### Operator: run playbooks, change monitoring
| Area | Tools |
|---|---|
| Automation | `run_playbook`, `cancel_ansible_run`, `create_schedule`, `update_schedule`, `delete_schedule`, `update_group_members` (put hosts into an inventory group or take them out; a group alert rule follows within a minute) |
| Monitors | `create_monitor`, `update_monitor`, `set_monitor_enabled` |
| Alerts | `create_alert_rule`, `update_alert_rule`, `acknowledge_alert`, `send_test_alert`, `create_maintenance_window`, `end_maintenance_window` |
| Metric anomalies | `review_anomaly` (mark one normal, so its hours are learned, or real) |
### Admin: destructive and remediation
`set_alert_action` (the playbook a rule runs when it fires), `delete_monitor`,
`delete_alert_rule`, `prune_docker`.
The client sees each tool's full contract: arguments, what an empty value
means, and what comes back. That contract is the tool's docstring, so read the
docstrings in `steward/mcp/tools/` for the detail.
---
## Response conventions
Every tool follows these conventions (`steward/mcp/tools/_shape.py`):
- **Times** are ISO-8601 UTC with a `Z`. Anything with a "when" also carries
`age_s`, so the client never does clock arithmetic.
- **Lists are bounded.** A list response carries `total` and `truncated`. When
it was cut, `more` names the argument that fetches the rest. Nothing is cut
silently.
- **Time series** come back as a summary (min/avg/max/p95/last) plus a series
of at most 60 points, never raw rows.
- **Lookup by name or id.** A miss suggests close matches and names the tool
that lists the valid choices.
- **Refusals are readable.** Bad input becomes a tool error that says what was
wrong, why, and what to do instead, and nothing is changed. The tools share
their validation with the web forms (`monitors/service.py`,
`alerts/service.py`), so a value the UI refuses is refused here for the same
reason.
- **Database outages** return a plain "Steward cannot reach its database"
rather than a stack trace.
---
## Adding tools
Core tools: add an `async def` to a module in `steward/mcp/tools/` and list it
in that module's `register()` with its minimum role and `read_only`. A plugin
exposes `register_mcp_tools(register)`, and optionally `mcp_overview(db)` to
contribute findings to `get_overview`. See
[Writing a plugin](../plugins/writing-a-plugin.md). A tool that changes
anything calls `audit_mcp(...)` from `steward/mcp/tools/_audit.py`.
docs/plugins/host-agent-design.md
+3 -19
View File
@@ -74,15 +74,8 @@ token = a1b2c3d4...
interval_seconds = 30
hostname = myhost # optional; defaults to uname -n
mounts = /, /mnt/data # optional; defaults to all real mounts (excluding tmpfs/devtmpfs/etc.)
docker_logs_enabled = true # optional; on by default — per-host container-log kill-switch
docker_log_exclude = watchtower, noisy-svc # optional; container names whose logs this host skips collecting
```
Container-log collection is on by default (no config needed). The two keys above
are the per-host opt-outs; the operator-facing global toggle + exclude live in
Settings and are enforced server-side (the push model has no channel to command
an agent), so they don't require touching a host's conf.
### Agent internals (function list, not classes)
- `read_config(path)` — parses the conf file into a dict.
@@ -92,17 +85,10 @@ an agent), so they don't require touching a host's conf.
- `collect_load()` — reads `/proc/loadavg`, returns `[1m, 5m, 15m]`.
- `collect_uptime()` — reads `/proc/uptime`, returns seconds since boot (int).
- `collect_metadata()` — `os.uname()` for kernel + arch, `/etc/os-release` for distro. Called once at startup and cached.
- `collect_docker_logs(socket, containers, state, exclude)` — (m79) per running
container, fetches new log lines over the Docker socket with an incremental
`since` cursor kept per container in `state` (a container's first interval
seeds from a short tail). Demuxes the multiplexed stream, parses each line's
RFC3339 timestamp, caps the whole batch at a byte limit (a marker line records
a truncation; the deferred lines come next interval). Folded into the sample as
`docker_logs`; omitted when empty.
- `build_payload()` — assembles a snapshot from all collectors into one dict.
- `post_payload(url, token, payloads)` — POSTs a list of samples, returns success/failure.
- `RingBuffer(maxlen=20)` — tiny FIFO wrapper, drops oldest when full.
- `main_loop()` — the 30s loop: collect → try POST → on failure push to buffer + backoff → on success flush buffer. Container logs are stripped from a sample before it's buffered (metrics survive an outage; stale logs are dropped).
- `main_loop()` — the 30s loop: collect → try POST → on failure push to buffer + backoff → on success flush buffer.
**Target: ~300 lines total including docstrings.** More than that is a smell that the agent is over-scoping.
@@ -157,6 +143,7 @@ plugins/host_agent/
│ └── host_agent_001_initial.py # creates host_agent_registrations table
├── templates/
│ ├── detail.html # per-host detail page
│ ├── widget_table.html # dashboard fleet table widget
│ ├── widget_history.html # dashboard history chart widget
│ ├── settings_list.html # plugin settings: list of registered hosts
│ └── install.sh.j2 # install script template (Jinja)
@@ -205,8 +192,6 @@ One `ScheduledTask` running every 60 seconds that flags `HostAgentRegistration`
**Note:** the existing alerts system can fire on "no metric for N seconds" which is arguably a cleaner signal. The scheduled task may end up redundant once alert rules exist. Build it anyway as a minimal safety net; it's one query plus one update.
**As built:** the task records `host_agent/<host>/seconds_since_seen` each minute, and "Agent offline" is an ordinary alert rule on it (`> 180`). It counts from the later of `last_seen_at` and the agent's newest post to *arrive* (kept in memory, `scheduler.arrivals`). `last_seen_at` is written in the ingest's own transaction, so a post Steward received but couldn't store (its storage stalled, #5514) would otherwise read as the agent's silence. Nothing counts from before Steward last started watching without a break (#5441).
### `METRIC_CATALOG` registration
Add to `steward/alerts/routes.py`:
@@ -270,7 +255,6 @@ Content-Type: application/json
- **`metadata` is sent on every POST**, not just on change. Server-side diff detects actual changes and only writes on change. Cost per POST is one dict — negligible. Benefit: server can cleanly detect agent restarts.
- **Raw bytes, not percentages, for memory and storage.** Percentages are derived server-side. Changing the "what counts as used" math doesn't require re-releasing the agent.
- **CPU is the one exception** — reported as a percentage because it's inherently a derivative (delta over time), not a snapshot. The agent must sample twice to compute it.
- **Container logs (m79) ride in the same push** as `docker_logs`: a list of `{container, stream, ts, line}` records — incremental since the previous interval. This pushes the *same direction* as metrics, so batched log history needs no inbound channel (only sub-second live-follow would). The server ingests them into `docker_logs`, enforces the global toggle + exclude list on ingest, and bounds storage with a per-container size+age ring. `docker_logs` is omitted when there's nothing new.
### Server expansion into `PluginMetric` rows
@@ -439,7 +423,7 @@ echo "Logs: journalctl -u steward-agent -f"
### Dashboard widgets (`steward/core/widgets.py`)
- The fleet glance (CPU, mem, disk /, load 1m, stale mark) is the core **Hosts — Overview** widget (`hosts_overview`); the old `host_resources` table was folded into it (#5213).
- **`host_resources`** — table widget. One row per monitored host: name, CPU %, mem %, disk % (worst mount), load 1m, "last seen Xs ago" with red/yellow/green coloring. Fleet glance.
- **`host_resource_history`** — chart widget for one host. CPU / mem / disk over a selectable time range (1h, 6h, 24h, 7d). Same pattern as every other history widget — Chart.js.
### Per-host detail page
docs/plugins/index.yaml.example
+113
View File
@@ -0,0 +1,113 @@
# steward-plugins / index.yaml
#
# This file is the catalog index for the steward plugin repository.
# It is fetched by the app's Settings → Plugins → Plugin Catalog UI.
#
# Steward reads this file from:
# https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml
#
# After adding or updating a plugin entry, commit and push — the change is
# live immediately for anyone whose app fetches the catalog (cache TTL: 5 min).
#
# REQUIRED fields for each plugin entry:
# name - must match the plugin's directory name exactly
# version - semver string
# download_url - direct URL to a zip file containing the plugin
# checksum_sha256 - SHA-256 hex digest of the zip (leave empty to skip verification)
#
# OPTIONAL fields:
# description, author, license, min_app_version,
# repository_url, homepage, tags
#
# Generating a checksum:
# sha256sum traefik.zip
# shasum -a 256 traefik.zip (macOS)
#
# Download URL conventions:
# Gitea release assets (recommended):
# https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip
# Upload the zip as a release attachment in Gitea; paste the URL here.
# Source archive tarballs work too but release assets are preferred (smaller, plugin-only).
version: 1
updated: "2026-03-22"
plugins:
- name: http
version: "1.0.0"
description: "Synthetic HTTP endpoint monitoring — status code, response time, content match, TLS expiry"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/http"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/http-v1.0.0/http.zip"
checksum_sha256: ""
tags:
- monitoring
- http
- synthetic
- uptime
- name: docker
version: "1.0.0"
description: "Docker container status, resource usage, and restart tracking via Docker socket"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v1.0.0/docker.zip"
checksum_sha256: ""
tags:
- containers
- docker
- infrastructure
- name: traefik
version: "1.0.0"
description: "Traefik reverse proxy metrics and access log integration"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip"
checksum_sha256: "" # fill in after running: sha256sum traefik.zip
tags:
- proxy
- metrics
- access-log
- name: unifi
version: "1.0.0"
description: "UniFi Network controller integration — WAN health, devices, clients, DPI"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/unifi-v1.0.0/unifi.zip"
checksum_sha256: ""
tags:
- network
- unifi
- ubiquiti
- name: host_agent
version: "1.0.0"
description: "Remote Linux host resource monitoring via a lightweight Python push agent"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/host_agent"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/host_agent-v1.0.0/host_agent.zip"
checksum_sha256: ""
tags:
- host
- monitoring
- cpu
- memory
- storage
docs/plugins/prometheus-widgets.md
-210
View File
@@ -1,210 +0,0 @@
# Prometheus widget templates
The Prometheus dashboard widget draws one scrape target through a **template**: a short YAML file that says which of the target's series to show and how. Steward ships with one, **Fabled Archives**. You can write your own, paste in one someone shared, and share yours as a file.
A template is data, not code. It names series and lays them out in **blocks**, and Steward draws them. It cannot add a colour, a chart or a formula. Every widget it draws has the same shape:
1. **A verdict:** "All well.", or the worst thing and how many need a look ("2 need a look. Replication: FullNightly-Restic is running, 11h 53m overdue.").
2. **Each block in turn**, its problems first, saying why, then the healthy rest. After six healthy rows a block says "+N more healthy".
3. **The target's name and when it was scraped**, linking to its page.
This page covers where templates live, a walk through a first template, each block with an example, the format reference, and the Fabled Archives template.
---
## Where templates live
**Prometheus → Widget templates** lists every template: the built-in ones, and your own. Open one to see its YAML beside a **preview**, which draws it over a target you pick from the values Steward has already stored. The preview redraws as you type and lists each problem with its line number.
| You want to | Do this |
|---|---|
| Start from scratch | **New template**, then write or paste YAML. |
| Use one someone sent you | **New template**, then paste it or **upload** the `.yaml` file. Look at the preview, then **Save template**. |
| Change a built-in one | Open it, then **Duplicate**. Built-in templates are read-only and change with Steward, so you edit the copy. |
| Share yours | **Export** downloads the YAML as a file. |
| Use it on a dashboard | Edit the dashboard, **Add widget → Prometheus**, and pick the target and the template. |
A template saves only when it has no problems, and its `name` must differ from every other template's. A widget remembers which template it draws with, so renaming a template keeps its widgets. Deleting one leaves its widgets saying the template is gone. Your templates are in Steward's backups, and adding, changing or deleting one is recorded in the audit log.
Steward never fetches a template from anywhere. Templates arrive only by being pasted or uploaded.
---
## Your first template
Look at the target's page first (**Prometheus → the target**). It lists every series Steward stored, with its labels and newest value. A template names those series exactly as listed there.
```yaml
name: My service
version: 1
expects: myservice_
blocks:
- values:
title: Health
items:
- label: Queue depth
metric: myservice_queue_depth
unit: count
warn_above: 100
- label: Disk free
metric: myservice_disk_free_bytes
of: myservice_disk_size_bytes
warn_below: 10%
crit_below: 5%
- times:
items:
- label: Last good run
metric: myservice_last_success_timestamp_seconds
warn_after: 26h
crit_after: 2d
```
- `name` is what the widget is called, and what the template list and the widget picker show.
- `version` is the template's own version. Raise it when you change a template you have shared, so the people you sent it to can tell copies apart.
- `expects` is the start of the metric names it reads. If the target reports none, the widget says so ("is this the template for it?") instead of a column of blanks.
**Metric names as Steward stores them.** A gauge keeps its name. A counter is stored as a rate per second: `requests_total` becomes `requests_per_sec`. A histogram keeps only `_sum_per_sec` and `_count_per_sec`. See [How series are stored](prometheus.md#how-series-are-stored).
---
## The blocks
Every block takes an optional `title`. Without one, it gets the default title shown in its heading below.
### `states`: a state set, one row per thing (default title "States")
A state set has one series per possible state, with the current one at 1 and the rest at 0. Archives' `fabledarchives_backup_target_state{target, state}` is one. Each thing it describes becomes a row naming its current state.
```yaml
- states:
title: Backup targets
metric: fabledarchives_backup_target_state
name: target # the label that names each row
healthy: [fresh]
quiet: [disabled, waiting] # shown greyed, never a problem
warning: [late] # anything not listed is a problem
columns:
- {label: Last backup, metric: fabledarchives_backup_target_last_backup_timestamp_seconds}
- {label: Stored, metric: fabledarchives_backup_target_stored_bytes}
```
- A state in `healthy` is fine, in `quiet` is shown greyed, and in `warning` is amber. **A state in none of the lists is red.** So a state the service adds later shows up as a problem rather than slipping by as healthy.
- A row reporting no current state is a warning.
- `columns` add the row's other series beside its state. Each column series is matched to its row by having the same labels. Failing that, it is matched by the label `name` points at.
- **Deadlines:** `warn_past` and `crit_past` each name a `*_timestamp_seconds` metric per row. Once that time has passed, a row in a healthy state reads as a warning, or a problem for `crit_past` ("is running, 11h 53m overdue"). Quiet and failing rows keep their own reading. Use this when a service ranks a state like "running" above lateness, so a run that never ends would otherwise read as healthy.
### `values`: figures with their units (default title "Figures")
```yaml
- values:
title: Storage
items:
- label: Disk free
metric: fabledarchives_data_disk_free_bytes
of: fabledarchives_data_disk_size_bytes # shown as "16.2 TiB of 41.9 TiB", with a bar
warn_below: 10% # a share of `of`
crit_below: 5%
sparkline: true # the last day as small bars beside it
- label: Stored
metric: fabledarchives_person_stored_bytes
name: person # one row per person
where: {kind: files} # only the series with kind=files
```
- `warn_above`, `crit_above`, `warn_below` and `crit_below` take a number, or a share like `10%` when the item has `of`. The red thresholds are checked before the amber ones.
- Without `name`, an item must match exactly one series. If more match, the row says how many and asks you to narrow it with `where` or `name`.
- A series the target doesn't report reads "—", "not reported", as a warning.
- The sparkline's bars are scaled to that series' own peak over the last 24 hours.
### `times`: moments and how long ago (default title "Times")
```yaml
- times:
items:
- label: Last good run
metric: myservice_last_success_timestamp_seconds
warn_after: 26h
crit_after: 2d
```
The metric is a `*_timestamp_seconds`. The row reads "3h ago" (exact time on hover) and turns amber or red once that long has passed. A timestamp of 0 reads "never", which is a warning when the item has a threshold.
### `table`: one metric, two labels (default title "Table")
```yaml
- table:
title: Stored per person
metric: fabledarchives_person_stored_bytes
rows: person
columns: kind
```
Rows and columns are the values of the two labels, in alphabetical order, under one header row. A missing cell reads "—". Tables show figures only; they don't judge them.
### `info`: labels worth reading (default title "About")
```yaml
- info:
title: Running
metric: fabledarchives_build_info
labels: [version, channel, commit] # which labels, in this order; all of them when left out
```
This block is for a `*_info` or `*_build_info` series, where the labels are the point and the value is always 1. It reads "version 2026.10.12.1717 · channel dev · commit 3605d0a".
---
## Reference
**Top level**
| Key | Needed | What it is |
|---|---|---|
| `name` | yes | At most 80 characters; unique among your templates and the built-in ones. |
| `version` | yes | A whole number from 1. |
| `blocks` | yes | One to 20 blocks. |
| `description` | | A line shown in the template list. |
| `expects` | | The prefix of the metric names it reads, like `fabledarchives_`. |
| `format` | | The template format; `1`, the only one so far. |
**Keys most blocks and items take**
| Key | What it is |
|---|---|
| `metric` | A series name as Steward stores it. |
| `where` | Labels a series must have, like `{kind: files, state: ok}`. |
| `name` | On `states`, `values` and `times`: the label that names each row (one row per value). |
| `unit` | On a `values` item, a `states` column or a `table`: overrides the unit read from the name: `bytes`, `seconds` (a span), `percent`, `per_sec`, `count` (thousands separated: 100,462), `timestamp`. |
Without `unit`, the name decides how a value reads: `_bytes` as bytes, `_seconds` as a span, `_timestamp_seconds` as a moment, `_per_sec` as a rate, `_pct` as a percentage. Anything else is a plain number.
**Spans** (`warn_after`, `crit_after`) are a number of seconds or a number with `s`, `m`, `h` or `d`: `90`, `15m`, `26h`, `2d`.
**Limits:** a template is at most 64 KB, with at most 20 blocks and 30 items in a block.
**What the checker refuses,** each with its line and what to change:
- an unknown key;
- a key given twice;
- anchors and aliases (`&name`, `*name`): write the value out;
- a share threshold (`10%`) without `of`;
- a state in two of `healthy`, `quiet` and `warning`;
- `crit_after` earlier than `warn_after`;
- a word YAML reads as true or false (`on`, `off`, `yes`, `no`) used as a state or label value. Quote it: `'off'`.
---
## The Fabled Archives template, walked through
The built-in template is a worked example of most of the above. **Duplicate** it to read or change it. It was written against the series Archives actually reports. On 14 Oct 2026 that turned up a nightly replication job that had read "running" for two days without a good copy; the deadlines below exist to catch exactly that.
| Block | What it shows | Why it is written this way |
|---|---|---|
| Backup targets (`states`) | Each target Archives receives backups from: fresh, late or missing, with its last backup and what it has stored. | `disabled` and `waiting` are quiet: a target you turned off, or one that hasn't sent its first backup, isn't a problem. `late` is a warning; `missing` (not listed) is a problem. |
| Replication (`states`) | Each replication job, its last good copy and its next run. | Archives ranks `running` above lateness, so `warn_past` and `crit_past` read the job's own late and missing times. A run that never finishes turns amber, then red. |
| Storage and integrity (`values`) | Disk free of its size (amber under 10%, red under 5%, last day as bars), files checked of files, malware held, failing notification channels. | Held malware and a failing channel each need you to do something, so any count above 0 is a warning. |
| Stored per person (`table`) | Each person's files, versions and trash. | Figures to read, not to judge. |
| Running (`info`) | The version, channel and commit. | |
Left out on purpose (the template's header says so too):
- `fabledarchives_section_up`: a section Archives couldn't read stops reporting its figures, and the rows that read them then say "not reported".
- The background queue, HTTP requests and audit counters: these suit charts and alert rules better than a glance.
docs/plugins/prometheus.md
-111
View File
@@ -1,111 +0,0 @@
# Prometheus Plugin
The Prometheus plugin scrapes any service's Prometheus `/metrics` endpoint and stores what you choose to keep in Steward's metrics. Once stored, a series can be charted, read through the MCP `query_metrics` tool, and watched by an alert rule like any other metric.
It is on by default. Targets are added, tested and edited on its page, **Prometheus** in the sidebar (`/plugins/prometheus/`).
---
## Adding a target
| Field | What it is |
|---|---|
| Name | How the target's series are named in Steward (rule pickers, charts). No `/`, `:` or comma. |
| Metrics address | The full URL, e.g. `http://archives:8081/metrics`. |
| Bearer token | Sent as `Authorization: Bearer …`. Stored encrypted and never shown again; leave it blank to keep the saved one. |
| Scrape every | 10 s to an hour. Default 60 s. |
| Check its HTTPS certificate | Off only for a self-signed certificate you trust. |
| Keep / Leave out | Name patterns, one a line, `*` for any run of characters. Empty Keep keeps everything; Leave out wins over Keep. |
| Series cap | The most series stored from one scrape (default 1000). Past it, the rest are counted and shown on the target's page, not stored. |
**Test** scrapes the address once with what is typed, filters included, and says how many series would be stored and under which names. Nothing is saved.
The endpoint hands over everything; Steward chooses what to keep. Start narrow (`yourservice_*`) and widen it if you find you want more: every stored series is a row every scrape.
---
## How series are stored
| On the endpoint | In Steward |
|---|---|
| A gauge or an untyped sample | Stored as it is, under its own name. |
| A counter (`requests_total`) | Stored as a rate per second, `requests_per_sec`, from the second scrape on. A value lower than the last (the service restarted) is skipped rather than read as a negative rate. |
| A histogram or summary | Its `_sum` and `_count` are stored as rates (`x_sum_per_sec`, `x_count_per_sec`). Buckets and quantiles are not stored. |
| NaN or ±Inf | Not stored. |
A series without labels is stored under the target's name. A labelled one is stored under `<target>:<label>=<value>,…`, in the order the endpoint gives the labels, for example `Archives:target=Steward,state=late`.
Each target also records four metrics about itself, under its own name:
| Metric | Meaning |
|---|---|
| `is_up` | 1 when the last scrape got metrics, 0 when it didn't. |
| `scrape_ms` | How long the scrape took. |
| `series_kept` | Series stored from the last scrape. |
| `series_capped` | Series the last scrape left out because of the cap. |
A value's unit is read from Prometheus naming: `_bytes` as bytes, `_seconds` as a span, `_timestamp_seconds` as a moment ("3h ago"), `_per_sec` as a rate.
A **state set** is one series per possible state, the current one 1 and the rest 0, such as `fabledarchives_backup_target_state{target, state}`. The target page shows it as one row per target (or job), naming the current state; the other states are listed on hover. A metric is read as a state set when its name ends in `_state` and every series has a `state` label reading 0 or 1. The OpenMetrics form, where the label is named after the metric (`foo{foo="a"} 1`), counts too. Each state is still stored as its own series, so an alert rule can watch one state.
**Only what the endpoint reports now is current.** When a scrape succeeds, any series it no longer has loses its newest value. That happens when the endpoint stopped reporting it, for example an old `build_info` after an upgrade, or when a filter or the cap now leaves it out. Its history stays on the chart until retention removes it. A failed scrape leaves the newest values as they were.
Renaming a target moves its stored history to the new name. Deleting one removes its newest values; its history ages out with retention (Settings → Retention).
---
## The dashboard widget
Edit a dashboard, open **Add widget**, and pick **Prometheus** under Integrations. Choose a **target** and a **template**; the widget is named after the template unless you give it a title.
A template says which of the target's series to show and how: backup states, figures with their units, times, a table, the version running. The widget leads with one line naming the worst problem, then each section with its problems first and the healthy rest after. Its **Details** link opens the target's page.
Steward ships with one template, **Fabled Archives**. A target whose series don't match its template says so in the widget rather than showing blanks.
## Widget templates
**Prometheus → Widget templates** lists every template the widget can draw with: the ones built in, and your own. A template of yours that no longer passes the checker (a newer Steward may read the format differently) is listed first with what to change, and the widgets drawn with it say so too.
Open one to see its YAML beside a **preview**: the template drawn over a target you choose, exactly as the widget draws it, from the values Steward has stored. Nothing is fetched from the target to draw it.
- **New template**: write one, or paste in one someone shared. The preview redraws as you type, and lists each problem with its line until there are none.
- **Upload a file**: on the same page, read a `.yaml` file into the editor to look over before saving.
- **Duplicate**: copy any template, built-in ones included, under a new name, and change the copy. Built-in templates are read-only: they ship with Steward and change with it.
- **Export**: download a template's YAML, to keep or to share.
How to write one, every block, and the format reference: [Prometheus widget templates](prometheus-widgets.md).
Saving needs a template that passes the checker and a name no other template has. A widget remembers which template it draws with, so renaming a template keeps its widgets; deleting one leaves its widgets saying the template is gone. Templates are kept in Steward's backups, and adding, changing or deleting one is in the audit log.
## Alerting on a target
In **Alerts → New rule**, choose **Prometheus targets**, then the series (the resource) and the metric. Two rules worth having for any target:
- **It stopped answering:** resource `<target>`, metric `is_up`, `< 1`.
- **Something was cut off:** resource `<target>`, metric `series_capped`, `> 0`.
---
## Recipe: Fabled Archives
Archives serves `/metrics` on its backup listener (port 8081), which is reachable only on your LAN and the swarm's overlay network, behind a scrape token.
1. In Archives, **Settings → Metrics → Make a token**, named e.g. "Steward". Copy it; it is shown once.
2. In Steward, **Prometheus → Add target**:
- Name `Archives`
- Metrics address `http://<archives host>:8081/metrics` (on a shared overlay network, the service name, e.g. `http://archives_app:8081/metrics`)
- Bearer token: the one you copied
- Keep: `fabledarchives_*`
3. **Test**, then **Add target**.
Rules worth adding:
| Watch | Resource | Metric | Condition |
|---|---|---|---|
| A backup target is late | `Archives:target=<name>,target_id=<id>,state=late` | `fabledarchives_backup_target_state` | `>= 1` |
| A backup target is missing | `Archives:target=<name>,target_id=<id>,state=missing` | `fabledarchives_backup_target_state` | `>= 1` |
| A replication job is failing | `Archives:job=<name>,job_id=<id>,state=failing` | `fabledarchives_replication_job_state` | `>= 1` |
| Archives' data disk is filling | `Archives` | `fabledarchives_data_disk_free_bytes` | `<` your floor |
| Archives stopped answering | `Archives` | `is_up` | `< 1` |
The target's page lists every stored series with its exact resource name, so copy names from there rather than typing them.
docs/plugins/traefik.md
+2 -12
View File
@@ -12,7 +12,7 @@ On each scrape:
3. Writes computed metrics to the `traefik_metrics` history table
4. Calls `record_metric()` for each metric, making them available to alert rules
Request rates and error rates are computed as deltas between the current and previous scrape, divided by elapsed time. Latency percentiles (p50, p95, p99) are approximated via linear interpolation over histogram buckets — these are estimates, not exact percentiles. They cover the responses since the previous scrape (each bucket's rise, summed over every status code and method), not every response since Traefik started. A scrape in which a router answered nothing records no latency, shown as "—"; one past the last bucket (5 s by default) reads as that bucket's edge.
Request rates and error rates are computed as deltas between the current and previous scrape, divided by elapsed time. Latency percentiles (p50, p95, p99) are approximated via linear interpolation over histogram buckets — these are estimates, not exact percentiles.
---
@@ -79,17 +79,7 @@ The widget auto-refreshes via HTMX polling every `poll_interval` seconds. Fragme
### Detail Page
The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time range picked at the top:
1. **The verdict.** It names the routers that need a look, says why, and says when the problem started ("api, 3.0% of responses are 5xx, started 2h ago"). When a router has been a problem for the whole range, it says so instead. Otherwise it reads "All N routers healthy".
2. **The request chart.** Requests per second across all routers and the share answered 5xx, the same chart as the dashboard widget.
3. **A card for each router that needs a look:** why, since when, the 4xx and 5xx status codes it answered in the last 15 minutes, most first ("404 ×1,812, 502 ×40"), and its request rate, bandwidth, p95/p99 latency and 4xx/5xx shares, each with its history. The code is what separates scanners probing for pages (404), a login wall (401/403) and an app that isn't answering (502/503/504). Steward records codes from the version that added them on; until then a card says they aren't recorded yet.
4. **Traefik's own figures:** when its config last reloaded, open connections, memory.
5. **The healthy routers,** folded into one table, then the routers Traefik no longer reports, then TLS certificates by soonest expiry, then Traffic Origin when the access log is on.
A router is graded on its last 15 minutes against Settings → Thresholds. When a problem started is read from the router's history over the range.
Once a router has 5 days of history, its 4xx share and p95 latency are judged against its own normal for that hour (see [Normals and anomalies](../architecture.md)) instead of the fixed cutoffs. One over the cutoff but within its normal is healthy, and the healthy table says it is usual for that router (an error-pages service that answers 4xx all day, a login wall's steady 401s). One out of its normal is a warning that says what is usual, and a metric anomaly to review; the Traefik page lists the routers' anomalies of the last 30 days. How far out counts is set on Settings → Thresholds → Normals (2 points of 4xx share, 50 ms of p95 by default). A 4xx share over fewer requests in the 15 minutes than Settings → Thresholds allows (30 by default) isn't judged at all. The 5xx share is a failure and always keeps its fixed cutoff. Latency is learned only from #5755's per-scrape figures, so its normal arrives 5 days after that release.
The full Traefik page at `/plugins/traefik/` shows all routers with sparkline history charts (last 20 data points) for request rate, p95 latency, and 5xx error rate.
---
docs/plugins/writing-a-plugin.md
+60 -59
View File
@@ -28,7 +28,7 @@ license: "MIT" # any SPDX identifier, e.g. MIT, Apache-2.0, GPL-3.0
# Optional: prevents loading on older app versions
min_app_version: "0.1.0"
# Optional: where the plugin's source lives
# Optional: shown in the catalog UI
repository_url: "https://github.com/yourname/yourrepo"
homepage: "https://github.com/yourname/yourrepo/tree/main/myplugin"
tags:
@@ -323,82 +323,83 @@ Role hierarchy: `admin > operator > viewer`. Requiring `viewer` grants access to
---
## Audit Log Entries
## Publishing to the Catalog
A change made through a plugin's pages is audited with `steward.core.audit.log_audit`, naming the thing it changed by `entity_type` and `entity_id`. So the audit page shows that thing's current name, linked to its page, export `get_audit_entities()` from `__init__.py`:
The official plugin catalog is hosted at `https://git.fabledsword.com/bvandeusen/Steward-plugins`. Anyone can submit a plugin by opening a pull request — first-party and third-party plugins are treated identically by the catalog system.
```python
# plugins/myplugin/__init__.py
def get_audit_entities() -> dict:
from .models import Target
return {"myplugin_target": (Target, "/plugins/myplugin/targets/{}")}
### Repo layout
```
steward-plugins/
├── index.yaml ← catalog index — the only file Steward fetches
├── myplugin/
│ ├── plugin.yaml
│ ├── __init__.py
│ └── ...
└── .github/
└── workflows/
└── publish.yml ← packages each plugin dir into a zip on release
```
The model needs `id` and `name` columns. An entity deleted since keeps the name the entry recorded in its detail (`detail={"name": ...}`). Add the action's wording to `ACTIONS` in `steward/audit/service.py` so the row reads as a sentence.
### index.yaml entry
---
Each plugin needs a corresponding entry in `index.yaml`. See `docs/plugins/index.yaml.example` for the full schema. The minimum required fields are:
## Agent Tools (MCP)
Steward serves an MCP endpoint at `/mcp` that AI clients reach with a personal API token created on the AI access page. A plugin adds tools to it by exporting an optional `register_mcp_tools(register)` from `__init__.py`:
```python
# plugins/myplugin/__init__.py
def register_mcp_tools(register) -> None:
from .mcp_tools import list_targets, rescan_target
from steward.models.users import UserRole
register(list_targets, role=UserRole.viewer, read_only=True)
register(rescan_target, role=UserRole.operator, read_only=False)
```yaml
- name: myplugin
version: "1.0.0"
description: "What this plugin does"
author: "Your Name"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://github.com/yourname/yourrepo"
homepage: "https://github.com/yourname/yourrepo/tree/main/myplugin"
download_url: "https://github.com/yourname/yourrepo/releases/download/myplugin-v1.0.0/myplugin.zip"
checksum_sha256: "" # fill in after zipping — see below
tags:
- monitoring
```
```python
# plugins/myplugin/mcp_tools.py
from steward.mcp._context import current_caller, mcp_app
### Creating a release zip
The zip must contain the plugin files at the top level OR inside a single directory named after the plugin. Both layouts work:
async def list_targets(limit: int = 50) -> dict:
"""What this tool is for, when to use it over its siblings, what each
argument's empty value means, and what comes back. This docstring is the
tool's description: it is all the agent knows about it."""
app = mcp_app() # Quart's current_app does not exist here
async with app.db_sessionmaker() as db:
...
return {"targets": [...], "total": n, "truncated": n > limit}
```
# Layout A — flat (preferred)
myplugin.zip
├── plugin.yaml
├── __init__.py
└── ...
# Layout B — single top-level directory (also accepted, GitHub archive default)
myplugin.zip
└── myplugin/
├── plugin.yaml
└── ...
```
- `role` and `read_only` are required. A tool is listed and callable only for callers whose role reaches `role`, and only while the plugin is enabled.
- Tools run outside Quart's request pipeline: use `mcp_app()` and `current_caller()` from `steward.mcp._context`, never `current_app` or `session`.
- Refuse by raising `ValueError("what was wrong, and what to do instead")`. The agent reads that text. Any other exception reaches it only as a generic failure.
- Tool names are global across core and every plugin; a name already taken fails registration and the plugin contributes no tools (logged).
- Hot-reloading the plugin replaces its tools.
- Follow the response conventions the core tools use (`steward/mcp/tools/_shape.py`): times via `iso()` plus `age_s()`, lists through `bounded()` so a cut is never silent, series through `summarize()`, and lookups by name or id through `pick()`.
Generate the zip and its checksum:
### Contributing to get_overview
`get_overview` is the agent's first call: what is wrong right now, ranked. A plugin adds its own problems to it by exporting an optional `mcp_overview(db)` that returns findings built with `steward.mcp.tools.overview.finding`:
```python
# plugins/myplugin/__init__.py
async def mcp_overview(db) -> list[dict]:
from steward.mcp.tools.overview import finding
return [
finding("warning", "target_unreachable", t.name, f"{t.name} has not answered for {t.misses} scans",
since=t.last_ok_at, ids={"target_id": t.id}, next=f'get_target(target="{t.id}")')
for t in await _unreachable_targets(db)
]
```bash
cd steward-plugins
zip -r myplugin.zip myplugin/
sha256sum myplugin.zip # paste this into index.yaml checksum_sha256
```
- `severity` is `"critical"` (something is down or about to be) or `"warning"` (degraded, or heading that way). Report problems only; a healthy plugin returns `[]`.
- It is called only while the plugin is enabled, with its own session. If it raises, the overview still returns, and names the plugin under `unavailable`.
Upload `myplugin.zip` as a GitHub release asset, then update `index.yaml` with the release download URL and checksum.
---
### How install works
## Shipping a Plugin
When a user clicks **Install** in Settings → Plugins:
Steward never downloads plugins. There is no catalog and no install button: the app runs with no outbound network.
1. The app downloads the zip from `download_url`
2. Verifies the SHA-256 checksum (if provided)
3. Extracts the plugin into its `PLUGIN_DIR`
4. Runs any pending Alembic migrations for the plugin
5. Attempts a **hot-reload** — registers the blueprint and scheduled tasks without restarting
6. If the plugin was previously loaded (blueprint already mounted), a **restart** is required to pick up the new code
- **First-party plugins** live in this repository under `plugins/` and ship inside the image, so they version with core.
- **A third-party plugin** is a folder you put in the external plugin root, `/data/plugins` by default (`STEWARD_PLUGIN_DIR` moves it). Restart Steward and the plugin appears in Settings → Plugins, where you enable it. A bundled plugin of the same name wins over an external one.
Hot-reload works reliably for brand-new plugin installs. Updates to already-active plugins require a restart, which can be triggered from the same settings page.
---
@@ -412,6 +413,6 @@ Steward never downloads plugins. There is no catalog and no install button: the
- [ ] Revision IDs prefixed with plugin name
- [ ] `record_metric()` called inside `session.begin()`
- [ ] Routes use `@require_role` decorator
- [ ] MCP tools (if any) registered via `register_mcp_tools(register)` with a role and `read_only`
- [ ] Templates namespaced under `templates/myplugin/`
- [ ] Plugin enabled in app settings
- [ ] `index.yaml` entry added with `download_url` and `checksum_sha256`
docs/reference/code-map.md
-23
View File
@@ -89,7 +89,6 @@ Quick reference for where key functions, models, and entry points live in the co
| `AlertState` | `steward/models/alerts.py` | `alert_states` |
| `AlertEvent` | `steward/models/alerts.py` | `alert_events` |
| `PluginMetric` | `steward/models/metrics.py` | `plugin_metrics` |
| `PluginMetricLatest` | `steward/models/metrics.py` | `plugin_metric_latest` |
| `AnsibleRun` | `steward/models/ansible.py` | `ansible_runs` |
| `User` | `steward/models/users.py` | `users` |
| `AppSetting` | `steward/models/settings.py` | `app_settings` |
@@ -137,25 +136,3 @@ Quick reference for where key functions, models, and entry points live in the co
| `steward/migrations/versions/` | Core schema (hosts, users, monitors, alerts, app_settings) |
| `plugins/traefik/migrations/versions/` | `traefik_metrics` table |
| `alembic.ini` | Alembic config; `version_locations` lists all migration directories |
---
## MCP server and API tokens
| What | File | Function / Class |
|---|---|---|
| Mount `/mcp` (ASGI dispatch, bearer auth, DB gate) | `steward/mcp/server.py` | `mount_mcp(app)` |
| Server, role registry, refusals | `steward/mcp/server.py` | `StewardMCPServer`, `register_tool()`, `refusal()` |
| Plugin tools and overview findings | `steward/mcp/server.py` | `register_plugin(name, module)` |
| Caller and app inside a tool | `steward/mcp/_context.py` | `current_caller()`, `mcp_app()` |
| Core tool modules | `steward/mcp/tools/` | `register_all()`; one module per area |
| Response conventions | `steward/mcp/tools/_shape.py` | `iso`, `age_s`, `bounded`, `lookback`, `summarize`, `pick` |
| Audit a write as "<user> via MCP" | `steward/mcp/tools/_audit.py` | `audit_mcp()` |
| "What is wrong now" | `steward/mcp/tools/overview.py` | `get_overview()` |
| Playbooks, runs, schedules, alert actions | `steward/mcp/tools/automation.py` | `run_playbook()`, `wait_for_run()`, … |
| Monitor validation (forms and tools) | `steward/monitors/service.py` | `monitor_fields()`, `build_config()` |
| Alert rule / window validation | `steward/alerts/service.py` | `rule_fields()`, `window_fields()`, `acknowledge()` |
| Docker and host-agent tools | `plugins/docker/mcp_tools.py`, `plugins/host_agent/mcp_tools.py` | `register_tools()`, `mcp_overview()` |
| Token model | `steward/models/api_tokens.py` | `ApiToken` |
| Mint, resolve, revoke tokens | `steward/core/api_tokens.py` | `create_token()`, `resolve_token()`, `revoke_token()` |
| Token management page and connect card | `steward/ai_access/routes.py`, `steward/templates/ai_access/` | `ai_access_bp` |
plugins/docker/__init__.py
+5 -36
View File
@@ -11,7 +11,7 @@ _app: "Quart | None" = None
def setup(app: "Quart") -> None:
global _app
_app = app
from .models import DockerContainer # noqa: F401 — register with Base
from .models import DockerContainer, DockerMetric # noqa: F401 — register with Base
# Publish the per-host persist hook so the host agent can hand off the
# `docker` array from each sample without importing our models (opportunistic
@@ -21,33 +21,21 @@ def setup(app: "Quart") -> None:
from steward.core.capabilities import register_capability
from steward.models.users import UserRole
from .ingest import persist_host_docker
from .retention import prune_docker_logs, run_docker_retention
from .retention import run_docker_retention
register_capability(
"docker.persist_host_samples", persist_host_docker,
label="Persist host Docker samples",
description="Store per-host container state + metrics pushed by the host agent.",
required_role=UserRole.viewer,
)
# Prune lifecycle events, driven by the core cleanup task without it
# importing our models. Same trusted server-side data-plane role as above.
# Container CPU and memory live in the one metric store and its retention.
# Roll up + prune Docker time-series, driven by the core cleanup task without
# it importing our models. Same trusted server-side data-plane role as above.
register_capability(
"docker.run_retention", run_docker_retention,
label="Run Docker retention",
description="Prune Docker lifecycle events past their window.",
description="Roll up old docker_metrics to hourly + prune stale metrics/events.",
required_role=UserRole.viewer,
)
register_capability(
"docker.prune_logs", prune_docker_logs,
label="Prune Docker logs",
description="Keep each container's logs within the age window and byte cap.",
required_role=UserRole.viewer,
)
# Each container's CPU (100% is one core) and memory share learn their
# normal (#5742), each with its floor from Settings → Thresholds.
from steward.core.norms import floor_setting, learn_normal
learn_normal("docker", "cpu_pct", floor_setting(app, "container_cpu_pct"))
learn_normal("docker", "mem_pct", floor_setting(app, "container_mem_pct"))
def get_scheduled_tasks() -> list:
@@ -61,25 +49,6 @@ def get_blueprint():
return docker_bp
def register_mcp_tools(register) -> None:
from .mcp_tools import register_tools
register_tools(register)
async def mcp_overview(db) -> list[dict]:
from .mcp_tools import mcp_overview as findings
return await findings(db)
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (fleet-wide container view).
return [{"label": "Docker", "href": "/plugins/docker/"}]
def get_alert_metrics() -> dict[str, list[str]]:
"""What ingest.py records per container (resource "<host>/<container>"):
restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 /
0.0 from the container's HEALTHCHECK, only for containers that define one.
Per host (resource "<host>"), what Docker keeps on its disk (#5304)."""
from .disk_view import DISK_METRICS
return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS]}
plugins/docker/disk_view.py
-82
View File
@@ -1,82 +0,0 @@
# plugins/docker/disk_view.py
"""What Docker keeps on each host's disk, over time and by image repository
(#5304–#5306). Model-free, so it is unit-tested without a database.
The agent reports `docker system df` as current state; ingest.py records its
totals as metrics too (DISK_METRICS, resource = the host's name), so each
host's growth has a history and an alert rule can watch it.
"""
from __future__ import annotations
from datetime import datetime
# metric name → the df summary field it records; "disk_total_bytes" is their sum.
DISK_PARTS = {
"disk_images_bytes": "images_size",
"disk_containers_bytes": "containers_size",
"disk_volumes_bytes": "volumes_size",
"disk_build_cache_bytes": "build_cache_size",
}
DISK_METRICS = [*DISK_PARTS, "disk_total_bytes", "disk_reclaimable_bytes"]
def disk_metrics(disk: dict) -> dict[str, float]:
"""The metric samples one df report gives: each part, their total, and
what unused images hold."""
out = {m: float(int(disk.get(f) or 0)) for m, f in DISK_PARTS.items()}
out["disk_total_bytes"] = sum(out.values())
out["disk_reclaimable_bytes"] = float(int(disk.get("images_reclaimable") or 0))
return out
def growth(points: list[tuple[datetime, float]]) -> dict | None:
"""How much a total changed over the points given (oldest first): the
change, the first point's time and value. None with fewer than two."""
if len(points) < 2:
return None
(first_at, first), (_, last) = points[0], points[-1]
return {"change": last - first, "since": first_at, "from": first, "to": last,
"pct": (last - first) / first * 100 if first else None}
def image_repo(repo_tag: str | None) -> tuple[str | None, str | None]:
"""("ghcr.io/x/app", "1.2") for "ghcr.io/x/app:1.2"; a registry's port is
part of the repository ("reg:5000/app" has no tag); a digest reference
keeps its digest as the tag; (None, None) for an untagged image."""
if not repo_tag or repo_tag == "<none>" or repo_tag.startswith("<none>"):
return None, None
if "@" in repo_tag:
repo, _, digest = repo_tag.partition("@")
return repo, digest[:19]
head, sep, tag = repo_tag.rpartition(":")
if not sep or "/" in tag:
return repo_tag, None
return head, tag
def group_images(images: list[dict]) -> list[dict]:
"""Images folded under their repository, the most space first.
Each image: {repo_tag, image_id, size, shared_size, containers}. A group's
`own` counts each version's size less the layers it shares with other
images, so a repository is not charged twice for one base layer;
`reclaimable` is that, for the versions no container uses. Untagged images
form one group of their own (repo None)."""
groups: dict[str | None, dict] = {}
for im in images:
repo, tag = image_repo(im.get("repo_tag"))
g = groups.setdefault(repo, {"repo": repo, "versions": [], "own": 0, "size": 0,
"reclaimable": 0, "containers": 0})
own = max(int(im.get("size") or 0) - int(im.get("shared_size") or 0), 0)
used = int(im.get("containers") or 0)
g["versions"].append({"tag": tag, "image_id": im.get("image_id"), "size": int(im.get("size") or 0),
"shared": int(im.get("shared_size") or 0), "own": own, "containers": used})
g["own"] += own
g["size"] = max(g["size"], int(im.get("size") or 0))
g["containers"] += used
if not used:
g["reclaimable"] += own
for g in groups.values():
g["versions"].sort(key=lambda v: (v["containers"] == 0, -v["size"]))
g["unused"] = sum(1 for v in g["versions"] if not v["containers"])
return sorted(groups.values(), key=lambda g: (g["repo"] is None, -g["own"], g["repo"] or ""))
plugins/docker/ingest.py
+40 -168
View File
@@ -34,16 +34,12 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
"""Diff a fresh snapshot against stored per-container state → lifecycle events.
Pure function (no DB) so it's unit-testable. `old_state` maps name → the
previously stored {status, health, oom_killed, exit_code, restart_count}; `new_containers`
previously stored {status, health, oom_killed, exit_code}; `new_containers`
is the newest snapshot's list of container dicts. Returns a list of
(container_name, event, detail) tuples:
start — a container transitions into running (or a genuinely new
container appears already running)
die — also for a new container first seen already exited
non-zero: a task that crashed between two samples (#5049)
restart — Docker restarted it (its restart count went up); said
instead of a start, since that is what happened
stop — running → not-running, or a running container is removed
die — same as stop but with a non-zero exit code (abnormal)
oom — OOMKilled flips False→True
@@ -66,22 +62,14 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
old = old_state.get(name)
if old is None:
# Newly observed container: a start, or, when it is already down
# with a non-zero exit, the death that happened between samples.
# Newly observed container — only a start is meaningful (we have no
# prior state to diff a death against).
if new_running:
events.append((name, "start", c.get("image") or None))
elif new_status in _STOPPED_STATES and new_exit not in (None, 0):
events.append((name, "die", f"exit {new_exit}"))
continue
old_running = (old.get("status") or "").lower() == "running"
new_restarts = c.get("restart_count") or 0
old_restarts = old.get("restart_count") or 0
if new_restarts > old_restarts:
gained = new_restarts - old_restarts
events.append((name, "restart", f"restart #{new_restarts}" if gained == 1
else f"{gained} restarts, now #{new_restarts}"))
if new_running and not old_running and new_restarts <= old_restarts:
if new_running and not old_running:
events.append((name, "start", c.get("image") or None))
elif old_running and not new_running:
if new_exit not in (None, 0):
@@ -104,55 +92,6 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
return events
def _log_rows(log_batches, host_id: str):
"""Pure: flatten the agent's log batches → docker_logs row kwargs (no DB).
`log_batches` is a list of (recorded_at, records) where each record is
{"container", "stream", "ts", "line"}. A line's own Docker timestamp wins;
recorded_at is the fallback when the agent couldn't parse one. Drops the
agent's advisory truncation marker (container "_steward" — it signals a
deferral, not a real container), malformed records, and lineless records;
normalises an unknown stream to stdout. Unit-testable in isolation.
"""
for recorded_at, records in log_batches:
if not isinstance(records, list):
continue
for rec in records:
if not isinstance(rec, dict):
continue
name = rec.get("container")
line = rec.get("line")
if not name or name == "_steward" or line is None:
continue
ts = _parse_started_at(rec.get("ts")) or recorded_at
stream = rec.get("stream")
if stream not in ("stdout", "stderr"):
stream = "stdout"
yield {"host_id": host_id, "container_name": str(name)[:255],
"ts": ts, "stream": stream, "line": str(line)}
async def _persist_logs(session, host, log_batches) -> None:
"""Append pushed container log lines (one row per line) for this host.
Time-series / append-only — the per-container size+age ring (retention)
bounds growth, so a chatty container just keeps a shorter window. The global
toggle + exclude list (Settings, no restart) are enforced here: this is the
authoritative drop point, since the push model has no channel to tell an
agent to stop collecting.
"""
from steward.core.settings import get_setting
from .models import DockerLog
if not await get_setting(session, "docker.logs.enabled"):
return
exclude = set(await get_setting(session, "docker.logs.exclude") or ())
for row in _log_rows(log_batches, host.id):
if row["container_name"] in exclude:
continue
session.add(DockerLog(**row))
async def _persist_swarm(session, host, swarm: dict) -> None:
"""Upsert this manager's swarm topology; drop rows no longer reported.
@@ -181,7 +120,6 @@ async def _persist_swarm(session, host, swarm: dict) -> None:
row.running = int(s.get("running") or 0)
row.image = s.get("image")
row.placement_json = json.dumps(s.get("placement") or [])
row.tasks_json = json.dumps(s.get("tasks") or [])
row.updated_at = now
stale_services = delete(DockerSwarmService).where(
DockerSwarmService.host_id == host.id)
@@ -240,14 +178,6 @@ async def _persist_disk(session, host, disk: dict) -> None:
summary.containers_count = int(disk.get("containers_count") or 0)
summary.volumes_count = int(disk.get("volumes_count") or 0)
summary.updated_at = now
# The totals as metrics too (#5304): a history of each host's growth, and
# something an alert rule can watch. Resource: the host's name.
from steward.core.alerts import MetricBatch
from .disk_view import disk_metrics
batch = MetricBatch()
for metric, value in disk_metrics(disk).items():
batch.add(source_module="docker", resource_name=host.name, metric_name=metric, value=value)
await batch.record(session)
images = disk.get("images") or []
seen: set[str] = set()
@@ -271,101 +201,43 @@ async def _persist_disk(session, host, disk: dict) -> None:
await session.execute(stale)
async def _persist_status(session, host, status: dict) -> None:
"""Record whether Docker answered on this host, and why not.
A host reporting "unavailable" with no row yet never had Docker: nothing to
record. Once a host has had Docker, losing it is recorded, so its last
containers read as stale rather than as running."""
from .models import DockerHostStatus
row = await session.get(DockerHostStatus, host.id)
if row is None:
if status["status"] == "unavailable":
return
row = DockerHostStatus(host_id=host.id)
session.add(row)
if row.status != status["status"]:
row.since = status["at"]
row.status = status["status"]
row.detail = (status.get("detail") or "")[:255] or None
row.reported_at = status["at"]
async def _expire_swarm(session, host) -> None:
"""Drop the swarm rows a host reported while it was a manager."""
from .models import DockerSwarmNode, DockerSwarmService
await session.execute(delete(DockerSwarmService).where(DockerSwarmService.host_id == host.id))
await session.execute(delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id))
async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
logs=None, status=None, phases=None) -> None:
async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -> None:
"""Upsert containers + time-series + lifecycle events + swarm for one host.
`snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
one entry per ingested sample carrying docker data (usually one; more when
the agent flushes a backlog). Every snapshot's running containers record
cpu_pct, mem_pct and mem_usage_bytes into the one metric store, at the
snapshot's time; the newest snapshot drives current container state and
lifecycle-event derivation. `swarm` is the newest
the agent flushes a backlog). Every snapshot contributes time-series
DockerMetric rows; the newest snapshot drives current container state, the
alert pipeline, and lifecycle-event derivation. `swarm` is the newest
sample's swarm object (or None off managers) — persisted when present.
`disk` is the newest sample's /system/df summary (or None on Docker-less
hosts) — persisted when present. `logs` is a list of (recorded_at, records)
log batches (m79) — appended one row per line when present, independent of
whether this sample also carried container metrics. `status` is the newest
sample's {"status", "detail", "at", "swarm_manager"} (agent ≥ 1.9.0): an
"ok" with an empty snapshot clears the host's containers, and a node that
says it is no longer a manager loses the swarm rows it reported as one.
`phases`, when given, is marked after each part (docker.status, .swarm,
.disk, .logs, .metrics, .events, .containers, .alerts, .reap), each
flushed before its mark, so a slow ingest names the part (#5181).
hosts) — persisted when present.
"""
from steward.core.alerts import MetricBatch
from .models import DockerContainer, DockerEvent
from steward.core.alerts import record_metric
from .models import DockerContainer, DockerEvent, DockerMetric
async def mark(name: str) -> None:
if phases is not None:
await session.flush()
phases.mark(f"docker.{name}")
if status is not None:
await _persist_status(session, host, status)
if status.get("swarm_manager") is False:
await _expire_swarm(session, host)
await mark("status")
if swarm is not None:
await _persist_swarm(session, host, swarm)
await mark("swarm")
if disk is not None:
await _persist_disk(session, host, disk)
await mark("disk")
if logs:
await _persist_logs(session, host, logs)
await mark("logs")
if not snapshots:
return
ordered = sorted(snapshots, key=lambda s: s[0])
latest_at, latest_containers = ordered[-1]
# Time series for every snapshot (running containers with a CPU read), in
# the one metric store (#5752): resource "<host>/<container>", so
# containers of the same name on different hosts don't collide. Recorded
# with the alert batch below, at each snapshot's own time.
batch = MetricBatch()
# Time-series points for every snapshot (running containers with a CPU read).
for recorded_at, containers in ordered:
for c in containers:
if c.get("status") != "running" or c.get("cpu_pct") is None or not c.get("name"):
continue
resource = f"{host.name}/{c['name']}"
for metric in ("cpu_pct", "mem_pct", "mem_usage_bytes"):
if c.get(metric) is not None:
batch.add(source_module="docker", resource_name=resource,
metric_name=metric, value=c[metric], recorded_at=recorded_at)
await mark("metrics")
if c.get("status") == "running" and c.get("cpu_pct") is not None:
session.add(DockerMetric(
host_id=host.id,
container_name=c["name"],
scraped_at=recorded_at,
cpu_pct=c["cpu_pct"],
mem_pct=c.get("mem_pct") or 0.0,
mem_usage_bytes=c.get("mem_usage_bytes") or 0,
))
# Snapshot of stored per-container state BEFORE the upsert overwrites it —
# used to diff lifecycle events. Skip event derivation on the host's first
@@ -375,24 +247,17 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
)).scalars().all()
old_state = {
r.name: {"status": r.status, "health": r.health,
"oom_killed": r.oom_killed, "exit_code": r.exit_code,
"restart_count": r.restart_count}
"oom_killed": r.oom_killed, "exit_code": r.exit_code}
for r in old_rows
}
if old_state:
# A stop is dated when Docker says it happened (agent ≥ 1.11), not when
# the sample that noticed it was taken.
finished = {c.get("name"): _parse_started_at(c.get("finished_at"))
for c in latest_containers}
for name, event, detail in _derive_events(old_state, latest_containers):
at = finished.get(name) if event in ("die", "stop") else None
session.add(DockerEvent(
host_id=host.id, container_name=name,
event=event, detail=detail, at=at or latest_at,
event=event, detail=detail, at=latest_at,
))
await mark("events")
# Current state, and the restart and health alerts, from the newest snapshot only.
# Current state + alerts from the newest snapshot only.
for c in latest_containers:
name = c.get("name")
if not name:
@@ -426,11 +291,23 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
existing.blk_read_bytes = c.get("blk_read_bytes")
existing.blk_write_bytes = c.get("blk_write_bytes")
# Alert pipeline — resource is host-scoped so containers of the same name
# on different hosts don't collide in the metric/alert namespace.
resource = f"{host.name}/{name}"
if c.get("status") == "running" and c.get("cpu_pct") is not None:
await record_metric(
session=session, source_module="docker",
resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
)
if c.get("mem_pct") is not None:
await record_metric(
session=session, source_module="docker",
resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
)
# Restart count is alertable regardless of state (crash-looping matters
# most while the container is down/restarting).
batch.add(
source_module="docker",
await record_metric(
session=session, source_module="docker",
resource_name=resource, metric_name="restart_count",
value=float(c.get("restart_count", 0) or 0),
)
@@ -439,16 +316,12 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
# container).
health = c.get("health")
if health in ("healthy", "unhealthy", "starting"):
batch.add(
source_module="docker",
await record_metric(
session=session, source_module="docker",
resource_name=resource, metric_name="is_healthy",
value=1.0 if health == "healthy" else 0.0,
)
await mark("containers")
await batch.record(session)
await mark("alerts")
# Reap containers that vanished from the host's latest listing. Without this,
# removed one-shot containers (CI job runners, buildkit builders, codex jobs)
# accumulate forever as permanent "stopped" rows and inflate every count.
@@ -461,4 +334,3 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
if seen_names:
reap = reap.where(DockerContainer.name.notin_(seen_names))
await session.execute(reap)
await mark("reap")
plugins/docker/mcp_tools.py
-676
View File
@@ -1,676 +0,0 @@
"""The docker plugin's MCP tools and its get_overview findings.
Registered through register_mcp_tools() in __init__.py, so they exist only
while the plugin is enabled. Reads go through service.py, the same functions
the plugin's pages use.
"""
from __future__ import annotations
import json
from dataclasses import dataclass
from datetime import datetime, timedelta
from sqlalchemy import select
from steward.ansible.runner import trigger_run
from steward.ansible.sources import BUILTIN_SOURCE_NAME
from steward.mcp._context import current_caller, mcp_app
from steward.mcp.tools._audit import audit_mcp
from steward.mcp.tools._shape import (
age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, summarize,
)
from steward.mcp.tools.overview import finding
from steward.models.ansible_inventory import AnsibleTarget
from steward.models.hosts import Host
from steward.models.users import UserRole
from . import service as docker_svc
from .service import PROBLEM_SAYS, PROBLEM_SEVERITY, PROBLEMS, container_problems, problem_since
from .dedup import dedup_by_container_id
from .models import DockerContainer, DockerDiskUsage, DockerSwarmNode, DockerSwarmService
from .swarm_view import build_swarm_services, replaced_tasks, task_time
CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
LOG_LINE_CHARS = 2000
# ── pure helpers ─────────────────────────────────────────────────────────────
def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
"""Keep both ends of an over-long line and say how much of the middle went."""
if text is None or len(text) <= limit:
return text
keep = limit // 2
return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
def container_row(c, host_names: dict, problems: list[str], now: datetime,
replaced: bool = False, since: datetime | None = None) -> dict:
running = (c.status or "").lower() == "running"
return {
"container_name": c.name,
"host_id": c.host_id,
"host_name": host_names.get(c.host_id),
"image": c.image,
"status": c.status,
"health": c.health,
"problems": problems,
# When the worst problem began, from the container's retained events.
"problem_since": iso(since),
"problem_age_s": age_s(since, now),
"exit_code": c.exit_code,
"oom_killed": bool(c.oom_killed),
"restart_count": c.restart_count,
"cpu_pct": rnd(c.cpu_pct),
"mem_pct": rnd(c.mem_pct),
"mem_usage_bytes": c.mem_usage_bytes,
"mem_limit_bytes": c.mem_limit_bytes,
"started_at": iso(c.started_at),
"uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
"compose_project": c.compose_project,
"swarm_service": c.service_name,
# A swarm task Swarm has replaced: history, judged on its service.
"swarm_replaced": replaced if c.service_name else None,
"reported_at": iso(c.scraped_at),
"reported_age_s": age_s(c.scraped_at, now),
}
def _from_iso(text: str | None) -> datetime | None:
return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
def container_findings(rows: list[dict]) -> list[dict]:
out = []
for r in rows:
if not r["problems"]:
continue
worst = r["problems"][0]
where = f" on {r['host_name']}" if r["host_name"] else ""
detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
out.append(finding(
PROBLEM_SEVERITY[worst], f"container_{worst}", r["container_name"],
f"container {r['container_name']}{where} {' and '.join(PROBLEM_SAYS[p] for p in r['problems'])}{detail}",
since=_from_iso(r["problem_since"]),
ids={"host_id": r["host_id"], "container_name": r["container_name"]},
next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
return out
def task_error_text(err: dict | None) -> str:
"""Swarm's own account of a task's end, as one clause."""
if not err:
return ""
why = err.get("error") or err.get("message") or err.get("state") or "unknown"
code = f", exit {err['exit_code']}" if err.get("exit_code") not in (None, 0) else ""
where = f" on {err['node']}" if err.get("node") else ""
return f"{why}{code}{where}"
def service_findings(services: list[dict]) -> list[dict]:
"""Swarm services that are failing: crash-looping, under-replicated, or that
lost a task to an error in the last 24h. Each names Swarm's last task error."""
out = []
for s in services:
replicas = f"{s['running']} of {s['desired']} replicas running"
last = task_error_text(s.get("last_error"))
tail = f"; last task error: {last}" if last else ""
ids = {"service_name": s["name"]}
since = s.get("trouble_since")
if s.get("crash_looping"):
out.append(finding(
"critical", "swarm_service_crash_looping", s["name"],
f"swarm service {s['name']} keeps failing: {s['failures_1h']} tasks failed in the "
f"last hour, {replicas}{tail}", since=since, ids=ids,
next="list_swarm_services()"))
elif s["state"] != "healthy":
out.append(finding(
"critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
s["name"], f"swarm service {s['name']} has {replicas}{tail}",
since=since, ids=ids, next="list_swarm_services()"))
elif s.get("failures_24h"):
n = s["failures_24h"]
out.append(finding(
"warning", "swarm_task_failed", s["name"],
f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
f"24h and is now {replicas}{tail}", since=since, ids=ids,
next="list_swarm_services()"))
return out
# ── loading ──────────────────────────────────────────────────────────────────
async def _host_names(db) -> dict[str, str]:
return dict((await db.execute(select(Host.id, Host.name))).all())
def _crash_loop_deaths() -> int:
return docker_svc.crash_loop_deaths(mcp_app().config)
async def _replaced(db, containers) -> set[tuple]:
services = list((await db.execute(select(DockerSwarmService))).scalars())
return replaced_tasks(containers, services)
async def _rows(db, now: datetime) -> list[dict]:
"""Every container with its problems. A swarm task Swarm replaced has none:
its failure, if it was one, belongs to its service (service_findings)."""
host_names = await _host_names(db)
return [container_row(r["container"], host_names, r["problems"], now,
replaced=r["replaced"], since=r["since"])
for r in await docker_svc.ranked_containers(db, now, _crash_loop_deaths())]
async def _swarm(db, now: datetime):
containers = dedup_by_container_id(await docker_svc.list_containers(db))
services = list((await db.execute(select(DockerSwarmService))).scalars())
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
host_names = await _host_names(db)
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
built = build_swarm_services(containers, services, node_hostname, host_names,
now=now, crash_loop_deaths=_crash_loop_deaths())
return built["services"], nodes, host_names
async def mcp_overview(db) -> list[dict]:
"""get_overview findings: failing containers and under-replicated swarm services."""
now = now_utc()
rows = await _rows(db, now)
services, _, _ = await _swarm(db, now)
return container_findings(rows) + service_findings(services)
# ── tools ────────────────────────────────────────────────────────────────────
async def list_containers(host: str = "", status: str = "", problem: str = "",
compose_project: str = "", limit: int = 100) -> dict:
"""Docker containers across every host that runs a Steward agent with Docker.
host: a host's name or id (list_hosts); empty for all hosts.
status: Docker's state — "running", "exited", "restarting", "paused",
"created" or "dead"; empty for all.
problem: "any" for containers with something wrong, or one of
"restarting", "crash_looping" (died repeatedly within the last hour;
the count is set under Settings → Thresholds, default 3), "unhealthy"
(failing its healthcheck), "oom_killed", "exited_nonzero" (died with an
error in the last 24h; exit 137/143 without an OOM kill is a stop, not
an error); empty for no filter.
compose_project: only containers from this Compose project.
limit: default 100, max 500.
Containers with problems come first. Each row's `problems` lists what is
wrong, most severe first, and `problem_since` / `problem_age_s` say when
the worst of them began (null when Steward's retained events cannot say).
`reported_age_s` is how old the agent's last report is, so a large value
means the row may be stale.
A swarm task's container that Swarm has replaced (a rolling update, or a
retry after a failure) has `swarm_replaced: true` and no problems: Swarm
restarts a task as a new container, so failures are counted on the
service — list_swarm_services shows them.
"""
if problem and problem != "any" and problem not in PROBLEMS:
raise ValueError(f"problem {problem!r} is not 'any' or one of {', '.join(PROBLEMS)}.")
limit = clamp_limit(limit, 100, 500)
now = now_utc()
async with db_session() as db:
rows = await _rows(db, now)
if host:
hosts = list((await db.execute(select(Host))).scalars())
h = pick(host, hosts, kind="host", list_tool="list_hosts")
rows = [r for r in rows if r["host_id"] == h.id]
if status:
rows = [r for r in rows if (r["status"] or "").lower() == status.lower()]
if problem == "any":
rows = [r for r in rows if r["problems"]]
elif problem:
rows = [r for r in rows if problem in r["problems"]]
if compose_project:
rows = [r for r in rows if r["compose_project"] == compose_project]
rows.sort(key=lambda r: (not r["problems"], (r["status"] or "") != "running",
(r["host_name"] or "").lower(), r["container_name"]))
return bounded(rows, limit, key="containers",
more="Pass a larger `limit` (max 500), or filter by `host`, `status` or `problem`.")
@dataclass
class RemovedContainer:
"""A container Docker has removed, known only by its retained logs and events."""
host_id: str
name: str
last_seen: datetime
_REMOVED_HINT = ("It may have been removed: Swarm replaces a dead task with a new container under "
"a new name. list_containers(status=\"exited\") lists stopped containers, and "
"search_container_logs(host=...) searches a whole host.")
async def _pick_container(db, container: str, host: str, *, removed: bool = False):
"""The container named `container`: a live one, else (with `removed`) one
Docker has removed whose logs or events Steward still holds, returned as a
RemovedContainer. Exact names beat partial ones, and live beats removed."""
containers = await docker_svc.list_containers(db)
host_id = None
if host:
hosts = list((await db.execute(select(Host))).scalars())
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
containers = [c for c in containers if c.host_id == host_id]
want = (container or "").strip()
if not want:
raise ValueError("pass a container name; list_containers lists them.")
live = {(c.host_id, c.name) for c in containers}
def _one(found, what):
if len(found) > 1:
raise ValueError(f"{len(found)} hosts have {what} named {want!r}; pass `host` "
f"(one of: {', '.join(sorted(f[0] for f in found))}).")
return found[0] if found else None
same = [c for c in containers if c.name == want] or \
[c for c in containers if c.name.lower() == want.lower()]
if same:
return _one([(c.host_id, c) for c in same], "a container")[1]
if removed:
gone = [r for r in await docker_svc.retained_names(db, want, exact=True, host_id=host_id)
if (r[0], r[1]) not in live]
if gone:
return RemovedContainer(*_one(gone, "a removed container"))
# One more chance: a unique container whose name contains it (swarm task
# names look like "stack_svc.1.k2j…").
partial = [c for c in containers if want.lower() in c.name.lower()]
if len(partial) == 1:
return partial[0]
gone = []
if removed and not partial:
gone = [r for r in await docker_svc.retained_names(db, want, exact=False, host_id=host_id)
if (r[0], r[1]) not in live]
if len(gone) == 1:
return RemovedContainer(*gone[0])
names = sorted(c.name for c in partial)[:10] + [f"{r[1]} (removed)" for r in gone[:10]]
hint = f" Containers whose names contain it: {', '.join(names)}." if names else ""
services = sorted({c.service_name for c in partial if c.service_name})
if services:
hint += (f" Swarm services among them: {', '.join(services[:10])}; search_container_logs "
"takes a service's name and searches every task it has run.")
raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} "
f"{'' if names else _REMOVED_HINT + ' '}list_containers lists them.")
def _removed_note(c: RemovedContainer) -> str:
return (f"{c.name} no longer exists: Docker removed it (Swarm replaces a failed task with a new "
f"container). This is what Steward retained; it was last seen {iso(c.last_seen)}.")
async def get_container(container: str, host: str = "", range: str = "24h", events: int = 20) -> dict:
"""One container in depth: state, resources over time and lifecycle events.
container: the container's name, as list_containers shows. A unique part of
a name also works (swarm task names are long).
host: the host's name or id. Needed only when several hosts run a
container with this name.
range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
"7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
events: how many lifecycle events (start, restart, stop, die, oom, health_change)
to return, newest first (default 20, max 100).
A container Docker has since removed (a dead Swarm task, a `docker rm`)
still resolves by its exact name, or by a part of it that matches one
removed container, while Steward retains its logs or events: the reply
has `removed: true`, `last_seen_at`, its history and its events, and no
live state. A live container has `removed: false`.
"""
since, key = lookback(range)
events = clamp_limit(events, 20, 100)
now = now_utc()
async with db_session() as db:
c = await _pick_container(db, container, host, removed=True)
if isinstance(c, RemovedContainer):
return await _removed_container(db, c, since, key, events, now)
gone = (c.host_id, c.name) in await _replaced(db, await docker_svc.list_containers(db))
failures = (await docker_svc.recent_failures_by_container(
db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
host_names = await _host_names(db)
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)])
hour_ago = now - timedelta(hours=1)
recent = sum(1 for e in failures if e.at >= hour_ago)
try:
ports = json.loads(c.ports_json or "[]")
except ValueError:
ports = []
problems = [] if gone else container_problems(c, recent, bool(failures), _crash_loop_deaths())
began = problem_since(problems, failures, hour_ago, sick.get((c.host_id, c.name)))
return {
**container_row(c, host_names, problems, now, replaced=gone, since=began),
"removed": False,
"container_id": c.container_id,
"ports": ports,
"net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
"blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
"range": key,
"cpu_pct_history": summarize(cpu),
"mem_pct_history": summarize(mem),
"events": _event_rows(evs, now),
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
}
def _event_rows(evs, now: datetime) -> list[dict]:
return [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)}
for e in evs]
async def _removed_container(db, c: RemovedContainer, since, key: str, events: int,
now: datetime) -> dict:
"""get_container for a container Docker has removed: what Steward kept."""
host_names = await _host_names(db)
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
return {
"container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id),
"removed": True, "note": _removed_note(c),
"last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now),
"range": key,
"cpu_pct_history": summarize(cpu),
"mem_pct_history": summarize(mem),
"events": _event_rows(evs, now),
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")',
}
async def list_swarm_services(limit: int = 100) -> dict:
"""Docker Swarm services with their replicas per node, and the swarm's nodes.
`state` is "healthy" (all desired replicas running), "degraded" (some) or
"down" (none). `replicas` are the service's current tasks: each names its
host and container status, and one marked `ghost` runs on a node with no
Steward agent, so Steward knows only how many tasks are placed there.
Tasks Swarm replaced (rolling updates, retries) are left out and counted
in `previous_tasks`.
From Swarm's own task records (a manager's agent 1.8.0 or later;
`tasks_known: false` and nulls otherwise):
- `failures_1h` / `failures_24h`: tasks that failed or were rejected.
Swarm retries a failed task as a new one, so this, not any container's
restart count, is how a crash loop shows. Swarm keeps about 5 old tasks
per replica, which bounds the count.
- `crash_looping`: failures_1h reached the crash-loop count set under
Settings → Thresholds (default 3).
- `last_error`: the newest task that failed, was rejected, or carries an
error while Swarm retries it — `error` is Swarm's own text (e.g. "No
such image", "task: non-zero exit (1)"), with `exit_code`, `node`, `at`.
- `trouble_since` / `trouble_age_s`: when the current trouble began — a
crash loop's first failure in the hour, a short service's first failed
task in its failing slots, a recovered service's first failure in 24h.
Swarm keeps few old tasks, so a long outage reads as a lower bound.
Services with a problem come first. `nodes` lists every swarm node once,
with its role (manager/worker), availability and status.
limit: default 100, max 500.
"""
limit = clamp_limit(limit, 100, 500)
now = now_utc()
async with db_session() as db:
services, nodes, host_names = await _swarm(db, now)
seen, node_rows = set(), []
for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
if n.node_id in seen:
continue
seen.add(n.node_id)
node_rows.append({"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
"leader": n.leader, "availability": n.availability, "status": n.status,
"reported_by": host_names.get(n.host_id)})
node_rows.sort(key=lambda n: n["hostname"].lower())
order = {"down": 0, "degraded": 1, "healthy": 2}
def _rank(s):
return (not s["crash_looping"], order.get(s["state"], 3), not s["failures_24h"], s["name"])
def _error_row(err):
if err is None:
return None
at = task_time(err)
return {"error": err["error"], "message": err["message"], "state": err["state"],
"exit_code": err["exit_code"], "node": err["node"],
"at": iso(at), "age_s": age_s(at, now) if at else None}
rows = [
{
"service_name": s["name"], "mode": s["mode"], "state": s["state"],
"running": s["running"], "desired": s["desired"],
"crash_looping": s["crash_looping"],
"failures_1h": s["failures_1h"], "failures_24h": s["failures_24h"],
"last_error": _error_row(s["last_error"]),
"trouble_since": iso(s["trouble_since"]),
"trouble_age_s": age_s(s["trouble_since"], now),
"previous_tasks": s["previous_tasks"], "tasks_known": s["tasks_known"],
"replicas": [
{"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
{"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
"status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
"cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
for r in s["replicas"]
],
}
for s in sorted(services, key=_rank)
]
return {"nodes": node_rows, **bounded(rows, limit, key="services")}
async def search_container_logs(container: str = "", host: str = "", text: str = "",
range: str = "1h", stream: str = "", before: str = "",
context: int = 0, limit: int = 50) -> dict:
"""Search the container log lines Steward has retained, newest first.
container: one container's name (as list_containers shows), or a Swarm
service's name (as list_swarm_services shows) to search every task
container the service has run, live and removed, so a search reaches
back across deploys; each match then has the `slot` and `task_id` that
wrote it. Empty searches every container (narrow with `host`). A
container Docker has removed still works while its lines are retained:
the reply then has `container_removed: true` and a `note` saying when it
was last seen.
host: a host's name or id; empty for all hosts.
text: case-insensitive substring to match; empty matches every line.
range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are
kept per container up to a size and age limit, so older lines may be gone.
stream: "stdout" or "stderr"; empty for both.
before: an ISO time; returns only lines older than it. Pass a response's
`next_before` to page further back.
context: lines of surrounding log (0-5) to include with each match, from
the same container. Only with `container` set.
limit: matches to return, default 50, max 200.
Lines longer than 2000 characters keep both ends, and the middle is
replaced by a marker saying how much was cut.
"""
since, key = lookback(range)
limit = clamp_limit(limit, 50, 200)
context = max(0, min(int(context or 0), 5))
if context and not container:
raise ValueError("context needs `container`: it shows lines around a match within one container's log.")
if stream and stream not in ("stdout", "stderr"):
raise ValueError(f"stream {stream!r} is not 'stdout' or 'stderr'.")
before_dt = None
if before:
try:
before_dt = datetime.fromisoformat(before.replace("Z", "+00:00"))
except ValueError as exc:
raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc
async with db_session() as db:
host_id = name = removed = service = tasks = None
if host:
hosts = list((await db.execute(select(Host))).scalars())
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
if container:
service, tasks = await _as_service(db, container.strip(), host_id)
if service is None:
c = await _pick_container(db, container, host, removed=True)
host_id, name = c.host_id, c.name
removed = c if isinstance(c, RemovedContainer) else None
rows = await docker_svc.search_logs(db, since=since, host_id=host_id, name=name,
containers=tasks, text=text, stream=stream,
before=before_dt, limit=limit + 1)
more = len(rows) > limit
rows = rows[:limit]
host_names = await _host_names(db)
matches = []
for r in rows:
m = {"at": iso(r.ts), "container_name": r.container_name, "host_id": r.host_id,
"host_name": host_names.get(r.host_id), "stream": r.stream, "line": cut_middle(r.line)}
if service:
m["slot"], m["task_id"] = docker_svc.task_of(service, r.container_name)
if context:
pre, post = await docker_svc.log_context(db, r.host_id, r.container_name, r.ts, context)
m["before"] = [cut_middle(x.line) for x in pre]
m["after"] = [cut_middle(x.line) for x in post]
matches.append(m)
out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more}
if service:
out["service"] = service
out["tasks_searched"] = len(tasks)
if removed is not None:
out["container_removed"] = True
out["note"] = _removed_note(removed)
if removed.last_seen < since:
out["note"] += " That is before this range; pass a longer `range`."
if more:
# Full precision: `at` is cut to the second for reading, and a cursor
# cut the same way would skip lines logged earlier in that second.
out["next_before"] = rows[-1].ts.isoformat()
out["more"] = "More matches are older; pass next_before as `before`, or narrow with `text`."
return out
async def _as_service(db, want: str, host_id: str | None):
"""(service, its task containers) when `want` names a Swarm service rather
than a container (#5468): every task container Steward holds anything for,
live and removed, kept to `host_id` when given (none there reads as no
lines, not as a missing name). (None, None) when it is not a service, or
a container carries that exact name, which keeps precedence."""
if not want:
return None, None
exact = (await db.execute(select(DockerContainer.name).where(
DockerContainer.name == want).limit(1))).first()
if exact is not None:
return None, None
tasks = await docker_svc.service_containers(db, want)
if not tasks:
return None, None
return want, [(h, n) for h, n in tasks if host_id is None or h == host_id]
async def get_docker_disk_usage(host: str = "") -> dict:
"""Docker disk usage per host: images, containers, volumes and build cache, in bytes.
host: a host's name or id; empty for every host. `images_reclaimable_bytes`
is what pruning unused images would free. An admin can prune from
Steward's Docker → Disk page.
"""
async with db_session() as db:
summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
host_names = await _host_names(db)
if host:
hosts = list((await db.execute(select(Host))).scalars())
hid = pick(host, hosts, kind="host", list_tool="list_hosts").id
summaries = [s for s in summaries if s.host_id == hid]
rows = [
{
"host_id": s.host_id, "host_name": host_names.get(s.host_id),
"images_total": s.images_total, "images_active": s.images_active,
"images_bytes": s.images_size, "images_reclaimable_bytes": s.images_reclaimable,
"containers_count": s.containers_count, "containers_bytes": s.containers_size,
"volumes_count": s.volumes_count, "volumes_bytes": s.volumes_size,
"build_cache_bytes": s.build_cache_size,
"reported_at": iso(s.updated_at), "reported_age_s": age_s(s.updated_at),
}
for s in summaries
]
rows.sort(key=lambda r: -(r["images_reclaimable_bytes"] or 0))
return {"hosts": rows, "total": len(rows)}
PRUNE_BUILTIN_PLAYBOOK = "maintenance/docker_prune.yml"
_STOPPED = ("exited", "created", "dead")
def prune_preview(what: str, usage, stopped: int) -> dict:
"""What a prune of `what` would remove, from the host's latest disk report."""
out: dict = {}
if what in ("containers", "system"):
out["stopped_containers"] = stopped
if what == "images":
out["unused_images_bytes"] = usage.images_reclaimable if usage else None
if what == "system":
out["dangling_images"] = "yes (unused but tagged images stay)"
out["build_cache_bytes"] = usage.build_cache_size if usage else None
out["unused_networks"] = "yes"
return out
async def prune_docker(host: str, what: str, check: bool = True) -> dict:
"""Reclaim Docker disk on a host. Previews unless check=false.
host: the Steward host's name or id. It needs a linked Ansible target
(list_inventory shows the links); the prune runs through Ansible as
Steward's built-in docker_prune playbook.
what: "containers" removes stopped containers; "images" removes ALL
unused images, not only dangling ones; "system" is `docker system
prune -f`: stopped containers, dangling images, unused networks and
build cache. Volumes are never pruned.
check: true (default) starts nothing and returns `would_remove`, sized
from the host's latest Docker disk report. Ansible's own check mode
skips the prune commands, so a check-mode run would show nothing.
Pass false to run the prune.
Admin only, as on the Docker → Disk page. A real prune returns `run_id`
(follow it with wait_for_run) and is recorded in the audit log.
"""
what = (what or "").strip()
if what not in docker_svc.PRUNE_TARGETS:
raise ValueError(f"what {what!r} is not one of {', '.join(docker_svc.PRUNE_TARGETS)}.")
async with db_session() as db:
h = pick(host, (await db.execute(select(Host))).scalars(), kind="host", list_tool="list_hosts")
target = (await db.execute(
select(AnsibleTarget).where(AnsibleTarget.host_id == h.id))).scalars().first()
usage = await db.get(DockerDiskUsage, h.id)
stopped = sum(1 for c in await docker_svc.list_containers(db, h.id)
if (c.status or "").lower() in _STOPPED)
if target is None:
raise ValueError(f"host {h.name!r} has no linked Ansible target, so Steward cannot reach it "
"to prune. Link one under Ansible → Inventory. Nothing was run.")
preview = {
"host": h.name,
"what": what,
"ansible_target": target.name,
"would_remove": prune_preview(what, usage, stopped),
"disk_report_age_s": age_s(usage.updated_at) if usage else None,
}
if check:
return {**preview, "check": True,
"summary": f"Nothing was run. Pass check=false to prune {what} on {h.name}."}
caller = current_caller()
run, _src, err = await trigger_run(
mcp_app(), source_name=BUILTIN_SOURCE_NAME, playbook_path=PRUNE_BUILTIN_PLAYBOOK,
inventory_scope=f"steward:target:{target.id}",
params={"extra_vars_map": docker_svc.prune_extra_vars(what)},
triggered_by=caller.user_id, origin="mcp")
if err:
raise ValueError(f"the prune could not start: {err}.")
await audit_mcp("docker.prune.started", "host", h.name,
{"host_id": h.id, "what": what, "run_id": run.id})
return {**preview, "check": False, "run_id": run.id,
"next": f'wait_for_run(run_id="{run.id}")'}
def register_tools(register) -> None:
for fn in (list_containers, get_container, list_swarm_services,
search_container_logs, get_docker_disk_usage):
register(fn, role=UserRole.viewer, read_only=True)
register(prune_docker, role=UserRole.admin, read_only=False)
plugins/docker/migrations/env.py
+1 -1
View File
@@ -15,7 +15,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
from steward.models.base import Base
import steward.models # noqa: F401
from plugins.docker.models import DockerContainer # noqa: F401
from plugins.docker.models import DockerContainer, DockerMetric # noqa: F401
config = context.config
if config.config_file_name is not None:
plugins/docker/migrations/versions/docker_009_container_logs.py
-48
View File
@@ -1,48 +0,0 @@
"""Docker container logs table
Adds docker_logs — one row per container log line, pushed by the host agent and
folded into its metrics push. Time-series, host-scoped (container names are only
unique within a host). Bounded by a per-container size+age ring in retention, so
a chatty container keeps a shorter window rather than growing without limit.
Additive create_table + twin indexes (viewer lookup by (host, container, ts);
age-cutoff prune by ts).
Revision ID: docker_009_container_logs
Revises: docker_008_bigint_mem
Create Date: 2026-07-19
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_009_container_logs"
down_revision: Union[str, None] = "docker_008_bigint_mem"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"docker_logs",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("host_id", sa.String(length=36), nullable=False),
sa.Column("container_name", sa.String(length=255), nullable=False),
sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
sa.Column("stream", sa.String(length=8), nullable=False, server_default="stdout"),
sa.Column("line", sa.Text(), nullable=False, server_default=""),
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_docker_logs_host_id", "docker_logs", ["host_id"])
op.create_index("ix_docker_logs_container_name", "docker_logs", ["container_name"])
op.create_index("ix_docker_logs_host_container_time",
"docker_logs", ["host_id", "container_name", "ts"])
op.create_index("ix_docker_logs_ts", "docker_logs", ["ts"])
def downgrade() -> None:
op.drop_index("ix_docker_logs_ts", table_name="docker_logs")
op.drop_index("ix_docker_logs_host_container_time", table_name="docker_logs")
op.drop_index("ix_docker_logs_container_name", table_name="docker_logs")
op.drop_index("ix_docker_logs_host_id", table_name="docker_logs")
op.drop_table("docker_logs")
plugins/docker/migrations/versions/docker_010_swarm_tasks.py
-30
View File
@@ -1,30 +0,0 @@
"""Docker swarm service tasks column
Adds docker_swarm_services.tasks_json — each service's newest tasks as a
manager's agent (≥ 1.8.0) reports them: slot, node, desired and current state,
Swarm's error and message, exit code and time. Lets Steward tell a task Swarm
replaced from one that crashed, count a service's failures across task
retries, and say why its last task failed. Additive column; no DROP+recreate.
Revision ID: docker_010_swarm_tasks
Revises: docker_009_container_logs
Create Date: 2026-10-06
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_010_swarm_tasks"
down_revision: Union[str, None] = "docker_009_container_logs"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("docker_swarm_services",
sa.Column("tasks_json", sa.Text, nullable=False,
server_default="[]"))
def downgrade() -> None:
op.drop_column("docker_swarm_services", "tasks_json")
plugins/docker/migrations/versions/docker_011_host_status.py
-36
View File
@@ -1,36 +0,0 @@
"""Docker host status
Adds docker_host_status: per host, whether Docker answered at the agent's
last report (ok / unavailable / error) and why not. Lets Steward show a host
whose Docker went away as such, instead of its last containers as running
(#5152). Additive create_table.
Revision ID: docker_011_host_status
Revises: docker_010_swarm_tasks
Create Date: 2026-10-06
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_011_host_status"
down_revision: Union[str, None] = "docker_010_swarm_tasks"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"docker_host_status",
sa.Column("host_id", sa.String(length=36), nullable=False),
sa.Column("status", sa.String(length=16), nullable=False),
sa.Column("detail", sa.String(length=255), nullable=True),
sa.Column("since", sa.DateTime(timezone=True), nullable=False),
sa.Column("reported_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("host_id"),
)
def downgrade() -> None:
op.drop_table("docker_host_status")
plugins/docker/migrations/versions/docker_012_drop_metrics.py
-56
View File
@@ -1,56 +0,0 @@
"""Container CPU and memory move to the one metric store
Drops docker_metrics and docker_metrics_hourly (#5752). Container cpu_pct,
mem_pct and mem_usage_bytes are recorded into plugin_metrics as source
"docker", resource "<host>/<container>", and roll up with every other metric.
CPU and memory % were already recorded there for alerting, so their history
stays; memory in bytes starts fresh.
Revision ID: docker_012_drop_metrics
Revises: docker_011_host_status
Create Date: 2026-10-14
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_012_drop_metrics"
down_revision: Union[str, None] = "docker_011_host_status"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute("DROP TABLE IF EXISTS docker_metrics_hourly")
op.execute("DROP TABLE IF EXISTS docker_metrics")
def downgrade() -> None:
op.create_table(
"docker_metrics",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("host_id", sa.String(36), sa.ForeignKey("hosts.id", ondelete="CASCADE"),
nullable=False, index=True),
sa.Column("container_name", sa.String(255), nullable=False, index=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True),
sa.Column("cpu_pct", sa.Float, nullable=False),
sa.Column("mem_pct", sa.Float, nullable=False),
sa.Column("mem_usage_bytes", sa.BigInteger, nullable=False),
)
op.create_index("ix_docker_metrics_host_container_time", "docker_metrics",
["host_id", "container_name", "scraped_at"])
op.create_table(
"docker_metrics_hourly",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("host_id", sa.String(36), sa.ForeignKey("hosts.id", ondelete="CASCADE"),
nullable=False),
sa.Column("container_name", sa.String(255), nullable=False),
sa.Column("bucket", sa.DateTime(timezone=True), nullable=False),
sa.Column("cpu_pct", sa.Float, nullable=False),
sa.Column("mem_pct", sa.Float, nullable=False),
sa.Column("mem_usage_bytes", sa.BigInteger, nullable=False),
sa.Column("sample_count", sa.Integer, nullable=False),
sa.UniqueConstraint("host_id", "container_name", "bucket",
name="uq_docker_metrics_hourly_bucket"),
)
op.create_index("ix_docker_metrics_hourly_bucket", "docker_metrics_hourly", ["bucket"])
plugins/docker/models.py
+67 -66
View File
@@ -1,12 +1,13 @@
# plugins/docker/models.py
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from sqlalchemy import (
BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text,
UniqueConstraint,
)
from sqlalchemy.orm import Mapped, mapped_column
from steward.models.base import Base
from steward.core.ids import time_ordered_id
class DockerContainer(Base):
@@ -60,6 +61,69 @@ class DockerContainer(Base):
blk_write_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
class DockerMetric(Base):
"""Time-series CPU/memory per container — one row per sample per running
container, scoped to the reporting host."""
__tablename__ = "docker_metrics"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
)
container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
scraped_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
index=True,
)
cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
# BigInteger: per-container memory usage exceeds 2^31 bytes (~2.1 GB) routinely.
mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
# Per-container history lookups filter on (host_id, container_name) then sort
# by time — a composite index keeps the rows() sparkline queries cheap.
__table_args__ = (
Index("ix_docker_metrics_host_container_time",
"host_id", "container_name", "scraped_at"),
)
class DockerMetricHourly(Base):
"""Hourly rollup of docker_metrics — avg cpu/mem per container per hour.
Raw per-sample rows (~2880/container/day at 30s) are pruned beyond a short
window; before deletion they're aggregated here so multi-day history stays
cheap to store and query. One row per (host, container, hour bucket); the
unique constraint lets retention upsert idempotently if it re-runs before the
raw rows are deleted. `bucket` is the hour-truncated sample time.
"""
__tablename__ = "docker_metrics_hourly"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
)
container_name: Mapped[str] = mapped_column(String(255), nullable=False)
bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
__table_args__ = (
# One bucket per container per host — the conflict target for the
# idempotent rollup upsert; doubles as the history-query index.
UniqueConstraint("host_id", "container_name", "bucket",
name="uq_docker_metrics_hourly_bucket"),
Index("ix_docker_metrics_hourly_bucket", "bucket"),
)
class DockerEvent(Base):
"""Lifecycle events derived by diffing consecutive host snapshots.
@@ -75,14 +139,14 @@ class DockerEvent(Base):
__tablename__ = "docker_events"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=time_ordered_id
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
)
container_name: Mapped[str] = mapped_column(String(255), nullable=False)
event: Mapped[str] = mapped_column(String(16), nullable=False)
# start | restart | stop | die | oom | health_change
# start | stop | die | oom | health_change
detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
# e.g. "exit 137", "healthy→unhealthy", image on start
at: Mapped[datetime] = mapped_column(
@@ -99,43 +163,6 @@ class DockerEvent(Base):
)
class DockerLog(Base):
"""Container log lines pushed by the host agent — one row per line.
Time-series, scoped to the reporting host (container names are only unique
within a host, same identity as docker_containers). The agent tails each running
container incrementally and folds new lines into its metrics push; `ts` is the
line's own Docker timestamp. Bounded by a per-container size+age ring
(retention), so a chatty container just keeps a shorter window rather than
growing without limit.
"""
__tablename__ = "docker_logs"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=time_ordered_id
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
)
container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
ts: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
)
stream: Mapped[str] = mapped_column(String(8), nullable=False, default="stdout")
# stdout | stderr
line: Mapped[str] = mapped_column(Text, nullable=False, default="")
# Viewer filters on (host_id, container_name) and sorts by time; the ring
# prune walks the same key newest-first. A second index on `ts` alone serves
# the age-cutoff delete. Twin-index idiom, mirroring docker_events.
__table_args__ = (
Index("ix_docker_logs_host_container_time",
"host_id", "container_name", "ts"),
Index("ix_docker_logs_ts", "ts"),
)
class DockerSwarmService(Base):
"""A Swarm service as seen by a manager host: desired vs running replicas.
@@ -159,11 +186,6 @@ class DockerSwarmService(Base):
# A manager sees every task, so this captures cross-node placement that the
# local docker_containers rows (this host only) can't.
placement_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
# The service's newest tasks, newest first, as the manager's agent shaped
# them: [{"task_id", "slot", "node_id", "desired_state", "state", "message",
# "error", "exit_code", "at"}], JSON. Swarm's own record of which tasks it
# replaced and why each one stopped; "[]" from agents older than 1.8.0.
tasks_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
@@ -240,24 +262,3 @@ class DockerSwarmNode(Base):
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
)
class DockerHostStatus(Base):
"""Whether Docker answered on a host at its agent's last report (agent ≥ 1.9.0).
"ok" when it did; "unavailable" when there is no Docker to ask (no socket,
nothing listening); "error" when Docker is there and the call failed.
`detail` says why for the last two. While a host is not "ok" its container
rows are what Docker last said, not what is true now, and read as stale.
A host that never had Docker gets no row.
"""
__tablename__ = "docker_host_status"
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
)
status: Mapped[str] = mapped_column(String(16), nullable=False)
detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
# When the status last changed, and the agent's newest report.
since: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
reported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
plugins/docker/overview.py
-150
View File
@@ -1,150 +0,0 @@
# plugins/docker/overview.py
"""The Docker page's two tables, as plain rows (#5299, #5301): one row per
swarm service and one per standalone container, with the search, stack, host
and status filters and the column sort the page offers. Model-free, so it is
unit-tested without a database; routes.rows() feeds it.
A stack is how the operator thinks of a group of services. Swarm names a
stack's services "<stack>_<service>", so a service's stack is the part of its
name before the first "_"; a standalone container's is its compose project.
Stacks filter the list rather than section it, as host groups do on Hosts.
"""
from __future__ import annotations
from datetime import datetime
from .swarm_view import service_trouble
# Status filter values and their labels; NO_STACK picks rows with no stack.
DOCKER_STATES = {"problem": "Needs attention", "healthy": "Healthy", "stopped": "Stopped"}
NO_STACK = "__none__"
# Each sort and whether it starts descending: the busiest first; "up" starts
# with the shortest uptime, the thing that most recently restarted.
DOCKER_SORTS = {"status": False, "name": False, "stack": False, "cpu": True, "mem": True,
"up": False}
def stack_of(service_name: str) -> str | None:
"""The stack a swarm service was deployed in: "arr" for "arr_radarr";
None for a name with no "_" (a service created on its own)."""
stack, sep, rest = service_name.partition("_")
return stack if sep and stack and rest else None
def _seconds_up(started_at, now: datetime) -> float | None:
return None if started_at is None else max((now - started_at).total_seconds(), 0.0)
def service_row(s: dict, now: datetime) -> dict:
"""A built swarm service (swarm_view.build_swarm_services) as a table row.
CPU and memory add up the running replicas; "up" is the newest replica's
start, so a service that just restarted a task reads as just restarted."""
reals = [r for r in s["replicas"] if not r["ghost"]]
running = [r for r in reals if (r["status"] or "").lower() == "running"]
cpu = [r["cpu_pct"] for r in running if r.get("cpu_pct") is not None]
mem = [r["mem_bytes"] for r in running if r.get("mem_bytes") is not None]
starts = [r["started_at"] for r in running if r.get("started_at")]
started = max(starts) if starts else None
return {
"kind": "service", "name": s["name"], "stack": stack_of(s["name"]),
"mode": s["mode"],
"image": s.get("image") or next((r["image"] for r in reals if r.get("image")), None),
"running": s["running"], "desired": s["desired"], "state": s["state"],
"trouble": service_trouble(s),
"nodes": sorted({r["host"] for r in reals}, key=str.lower),
"ghost_nodes": sorted({r["host"] for r in s["replicas"] if r["ghost"]}, key=str.lower),
"cpu_cores": sum(cpu) / 100 if cpu else None,
"mem_bytes": sum(mem) if mem else None,
"started_at": started, "up_s": _seconds_up(started, now),
"unhealthy": any(r.get("health") == "unhealthy" for r in reals),
"failures_24h": s.get("failures_24h"), "crash_looping": s.get("crash_looping"),
"last_error": s.get("last_error"), "trouble_since": s.get("trouble_since"),
}
def container_trouble(c, *, stale: bool, exit_failed: bool) -> int | None:
"""How bad a standalone container is, on service_trouble's scale: 1
unhealthy or restarting, 2 died with a failing exit, 3 its report is
stale; None when nothing is wrong (a cleanly stopped one included)."""
if c.health == "unhealthy" or c.status == "restarting":
return 1
if c.status != "running" and exit_failed:
return 2
if stale:
return 3
return None
def container_row(c, *, label: str, host: str, usage: dict, ports: list, stale: bool,
exit_failed: bool, now: datetime) -> dict:
"""A standalone container as a table row; `usage` is service.usage()."""
running = c.status == "running"
return {
"kind": "container", "name": label, "container_name": c.name, "host": host,
"host_id": c.host_id, "stack": c.compose_project or None, "image": c.image,
"status": c.status, "health": c.health, "running": running, "ports": ports,
"usage": usage, "cpu_cores": usage.get("cores") if running else None,
"mem_bytes": usage.get("mem_bytes") if running else None,
"started_at": c.started_at if running else None,
"up_s": _seconds_up(c.started_at, now) if running else None,
"exit_code": c.exit_code, "oom_killed": c.oom_killed, "exit_failed": exit_failed,
"restart_count": c.restart_count or 0, "stale": stale, "scraped_at": c.scraped_at,
"trouble": container_trouble(c, stale=stale, exit_failed=exit_failed),
}
def _hosts(row: dict) -> list[str]:
return row["nodes"] + row["ghost_nodes"] if row["kind"] == "service" else [row["host"]]
def _stopped(row: dict) -> bool:
return (row["running"] == 0) if row["kind"] == "service" else not row["running"]
def filter_rows(rows: list[dict], *, q: str = "", stack: str = "", host: str = "",
state: str = "") -> list[dict]:
"""Rows matching a search (name, container name, image or stack, any
case), a stack (or NO_STACK), a host or node, and a Status filter; an
empty value doesn't filter."""
q = (q or "").strip().lower()
out = []
for r in rows:
if q and not any(q in (v or "").lower() for v in
(r["name"], r.get("container_name"), r["image"], r["stack"])):
continue
if (stack == NO_STACK and r["stack"]) or (stack and stack != NO_STACK and r["stack"] != stack):
continue
if host and host not in _hosts(r):
continue
if state == "problem" and r["trouble"] is None:
continue
if state == "healthy" and (r["trouble"] is not None or _stopped(r)):
continue
if state == "stopped" and not _stopped(r):
continue
out.append(r)
return out
def sort_rows(rows: list[dict], sort: str, desc: bool) -> list[dict]:
"""Rows in a column's order. Ties keep name order; a row with no value for
the column goes last whichever way it runs. "status" is problems first,
worst first, then running, then stopped."""
by_name = sorted(rows, key=lambda r: r["name"].lower())
if sort == "name":
return list(reversed(by_name)) if desc else by_name
if sort == "status":
return sorted(by_name, key=lambda r: (4 if r["trouble"] is None else r["trouble"],
_stopped(r)), reverse=desc)
value = {"stack": lambda r: (r["stack"] or "").lower() or None,
"cpu": lambda r: r["cpu_cores"], "mem": lambda r: r["mem_bytes"],
"up": lambda r: r["up_s"]}[sort]
have = [r for r in by_name if value(r) is not None]
return sorted(have, key=value, reverse=desc) + [r for r in by_name if value(r) is None]
def filter_options(services: list[dict], containers: list[dict]) -> dict:
"""The stack and host choices the filter bar offers, from every row."""
rows = services + containers
return {"stacks": sorted({r["stack"] for r in rows if r["stack"]}, key=str.lower),
"hosts": sorted({h for r in rows for h in _hosts(r)}, key=str.lower)}
plugins/docker/retention.py
+100 -67
View File
@@ -1,95 +1,128 @@
# plugins/docker/retention.py
"""Bound Docker history growth: prune old lifecycle events and log lines.
"""Bound Docker time-series growth: roll up old metrics, prune old rows.
Published as the "docker.run_retention" and "docker.prune_logs" capabilities
(see __init__.setup) so the core cleanup task can drive them WITHOUT importing
the docker models (same opportunistic-coupling pattern as
docker.persist_host_samples). Each runs inside the caller's open transaction;
never opens or commits its own.
Published as the "docker.run_retention" capability (see __init__.setup) so the
core cleanup task can drive it WITHOUT importing the docker models (same
opportunistic-coupling pattern as docker.persist_host_samples). Runs inside the
caller's open transaction; never opens or commits its own.
Container CPU and memory are not here: they live in the one metric store
(plugin_metrics) and its rollup and retention (#5752).
The scaling concern is docker_metrics: ~2880 rows/container/day at a 30s sample.
We keep raw samples for a short window, then aggregate everything older into
hourly averages (docker_metrics_hourly) and delete the raw rows — so multi-day
history stays cheap to store and query. docker_events is light but unbounded
without a cutoff, so it gets a (longer) window too.
"""
from __future__ import annotations
from datetime import datetime, timedelta
def _hour_floor(dt: datetime) -> datetime:
"""Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
return dt.replace(minute=0, second=0, microsecond=0)
def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
"""Hour-aligned boundary below which raw metrics get rolled up + deleted.
Aligning to the hour means we only ever roll up *whole* elapsed hours — a
bucket is never split across the keep/roll boundary, so re-running can't
produce a partial-then-complete duplicate for the same hour.
"""
return _hour_floor(now - timedelta(days=raw_days))
async def run_docker_retention(
session,
*,
events_days: int,
metrics_raw_days: int,
metrics_rollup_days: int,
now: datetime | None = None,
) -> dict:
"""Prune docker_events older than the events window. Returns a counts
dict for logging. The container-log ring is prune_docker_logs, run in its
own transaction."""
from datetime import timezone
"""Roll up + prune Docker time-series. Returns a counts dict for logging.
from sqlalchemy import delete
from .models import DockerEvent
if now is None:
now = datetime.now(timezone.utc)
res = await session.execute(
delete(DockerEvent).where(DockerEvent.at < now - timedelta(days=events_days))
)
return {"events_pruned": res.rowcount or 0}
async def prune_docker_logs(
session,
*,
retention_days: int,
max_bytes_per_container: int,
now: datetime | None = None,
) -> dict:
"""The docker_logs ring (m79): drop lines older than the age window, then
keep only the newest ~cap bytes per (host, container). Runs inside the
caller's open transaction, like run_docker_retention.
Only containers at or over the cap are ranked (#5449). Ranking every line
in the table each hour to find the few containers over it sorted the whole
table on slow storage.
1. Aggregate docker_metrics older than the (hour-aligned) raw window into
docker_metrics_hourly (avg cpu/mem per container per hour), upserting so a
re-run is idempotent, then delete those raw rows.
2. Prune rolled-up rows older than the rollup window.
3. Prune docker_events older than the events window.
"""
from datetime import timezone
from sqlalchemy import and_, delete, func, select
from sqlalchemy import delete, func, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from .models import DockerLog
from .models import DockerEvent, DockerMetric, DockerMetricHourly
if now is None:
now = datetime.now(timezone.utc)
res = await session.execute(
delete(DockerLog).where(DockerLog.ts < now - timedelta(days=retention_days))
)
age_pruned = res.rowcount or 0
rolled = rolled_rows = events_pruned = rollup_pruned = 0
size = func.length(DockerLog.line)
over = (
select(DockerLog.host_id, DockerLog.container_name)
.group_by(DockerLog.host_id, DockerLog.container_name)
.having(func.sum(size) >= max_bytes_per_container)
.subquery()
)
# Per container, sum line bytes newest-first; delete a row once its
# strictly-newer siblings already fill the cap. The EXCLUSIVE prefix
# (running total minus this row) means the newest row always survives, so
# a single line larger than the cap is never wiped out.
running = func.sum(size).over(
partition_by=[DockerLog.host_id, DockerLog.container_name],
order_by=[DockerLog.ts.desc(), DockerLog.id.desc()],
)
ranked = (
select(DockerLog.id, (running - size).label("prefix_excl"))
.join(over, and_(DockerLog.host_id == over.c.host_id,
DockerLog.container_name == over.c.container_name))
.subquery()
# ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
raw_cutoff = _rollup_cutoff(now, metrics_raw_days)
hour = func.date_trunc("hour", DockerMetric.scraped_at)
agg = (
select(
DockerMetric.host_id,
DockerMetric.container_name,
hour.label("bucket"),
func.avg(DockerMetric.cpu_pct).label("cpu_pct"),
func.avg(DockerMetric.mem_pct).label("mem_pct"),
func.avg(DockerMetric.mem_usage_bytes).label("mem_usage_bytes"),
func.count().label("sample_count"),
)
.where(DockerMetric.scraped_at < raw_cutoff)
.group_by(DockerMetric.host_id, DockerMetric.container_name, hour)
)
for r in (await session.execute(agg)).all():
stmt = (
pg_insert(DockerMetricHourly)
.values(
host_id=r.host_id,
container_name=r.container_name,
bucket=r.bucket,
cpu_pct=float(r.cpu_pct or 0.0),
mem_pct=float(r.mem_pct or 0.0),
mem_usage_bytes=int(r.mem_usage_bytes or 0),
sample_count=int(r.sample_count or 0),
)
.on_conflict_do_update(
constraint="uq_docker_metrics_hourly_bucket",
set_={
"cpu_pct": float(r.cpu_pct or 0.0),
"mem_pct": float(r.mem_pct or 0.0),
"mem_usage_bytes": int(r.mem_usage_bytes or 0),
"sample_count": int(r.sample_count or 0),
},
)
)
await session.execute(stmt)
rolled += 1
rolled_rows += int(r.sample_count or 0)
if rolled:
await session.execute(
delete(DockerMetric).where(DockerMetric.scraped_at < raw_cutoff)
)
# ── 2. Prune rolled-up rows beyond the rollup window ──
rollup_cutoff = now - timedelta(days=metrics_rollup_days)
res = await session.execute(
delete(DockerLog).where(DockerLog.id.in_(
select(ranked.c.id).where(ranked.c.prefix_excl >= max_bytes_per_container)))
delete(DockerMetricHourly).where(DockerMetricHourly.bucket < rollup_cutoff)
)
return {"logs_age_pruned": age_pruned, "logs_size_pruned": res.rowcount or 0}
rollup_pruned = res.rowcount or 0
# ── 3. Prune lifecycle events beyond the events window ──
events_cutoff = now - timedelta(days=events_days)
res = await session.execute(
delete(DockerEvent).where(DockerEvent.at < events_cutoff)
)
events_pruned = res.rowcount or 0
return {
"buckets_rolled": rolled,
"raw_rows_rolled": rolled_rows,
"rollup_pruned": rollup_pruned,
"events_pruned": events_pruned,
}
plugins/docker/routes.py
+282 -693
View File
File diff suppressed because it is too large Load Diff
plugins/docker/service.py
-750
View File
@@ -1,750 +0,0 @@
"""Docker queries shared by the plugin's pages and its MCP tools.
Each function takes an open session and returns rows or plain data, with no
Quart request context, so the routes and the /mcp tools (which run outside
Quart's pipeline) call the same code.
"""
from __future__ import annotations
import re
from datetime import datetime, timedelta, timezone
from sqlalchemy import func, select
from steward.core import metric_series
from steward.models.hosts import Host
from steward.models.metrics import PluginMetric
from .dedup import dedup_by_container_id
from .models import (
DockerContainer, DockerEvent, DockerLog, DockerSwarmService,
)
from .swarm_view import replaced_tasks
CRASH_LOOP_DEATHS = 3 # default; Settings → Thresholds sets it
def crash_loop_deaths(config) -> int:
"""Deaths within an hour that make a container or a swarm service a crash
loop, from Settings → Thresholds (app.config["THRESHOLDS"])."""
th = (config.get("THRESHOLDS") or {}).get("crash_loop") or {}
return int(th.get("warn") or CRASH_LOOP_DEATHS)
# Exit codes a deliberate stop leaves: SIGTERM (143), and SIGKILL (137) when
# the process outlives the stop timeout. Without an OOM kill they are someone
# stopping the container, not the container failing.
STOP_EXITS = (137, 143)
def exit_is_failure(exit_code, oom_killed: bool = False) -> bool:
"""Whether a container's exit is a failure: an OOM kill, or a non-zero
exit other than a stop's."""
return bool(oom_killed) or exit_code not in (None, 0, *STOP_EXITS)
def _event_exit(detail) -> int | None:
"""The exit code in a die event's detail ("exit 1"), or None."""
text = (detail or "").strip()
if text.startswith("exit "):
try:
return int(text[5:])
except ValueError:
return None
return None
def event_is_failure(e) -> bool:
"""Whether a die/oom event is a failure. An oom always is; a die is unless
its exit is a stop's (or clean)."""
if e.event == "oom":
return True
return e.event == "die" and exit_is_failure(_event_exit(e.detail))
def failures_24h(events, swarm_services, host_name: dict) -> list[dict]:
"""What failed in the last 24h, newest first, for the dashboard count and
the Docker page's list of it: one row per plain container that died or was
OOM-killed, and one per swarm service with failed tasks.
`events` are die/oom events of the last 24h. A swarm task's container dies
whenever Swarm replaces it, so a service with Swarm's task records counts
its failed tasks instead of its containers' deaths; a service without them
(an agent older than 1.8.0) falls back to its tasks' deaths.
Each row: {kind: "container"|"service", name, host_id, host, count, at,
detail, task_id}; task_id is the failed swarm task, so its logs can be
found (None for a container, or a service without task records).
"""
by_prefix = {s["name"] + ".": s for s in swarm_services}
rows: dict[tuple, dict] = {}
# Each manager reports every swarm task, so one task's death can arrive
# once per manager: a service counts each dead task container once.
task_deaths: dict[tuple, set] = {}
for e in sorted(events, key=lambda e: e.at, reverse=True):
if not event_is_failure(e):
continue
svc = next((s for p, s in by_prefix.items() if e.container_name.startswith(p)), None)
if svc is not None and svc["tasks_known"]:
continue
kind, name = ("service", svc["name"]) if svc else ("container", e.container_name)
key = (kind, name) if svc else (kind, e.host_id, name)
if svc is not None:
seen = task_deaths.setdefault(key, set())
if e.container_name in seen:
continue
seen.add(e.container_name)
row = rows.get(key)
if row is None:
rows[key] = {"kind": kind, "name": name, "host_id": e.host_id,
"host": host_name.get(e.host_id, e.host_id), "count": 1, "at": e.at,
"detail": "OOM-killed" if e.event == "oom" else e.detail,
"task_id": (task_of(svc["name"], e.container_name) or (None, None))[1]
if svc else None}
else:
row["count"] += 1
from .swarm_view import task_time
for s in swarm_services:
if s["tasks_known"] and s["failures_24h"]:
err = s["last_error"] or {}
rows[("service", s["name"])] = {
"kind": "service", "name": s["name"], "host_id": None, "host": err.get("node"),
"count": s["failures_24h"], "at": task_time(err) if err else None,
"detail": err.get("error") or err.get("message") or err.get("state"),
"task_id": err.get("task_id")}
epoch = datetime.min.replace(tzinfo=timezone.utc)
return sorted(rows.values(), key=lambda r: r["at"] or epoch, reverse=True)
def task_label(c, node_hostname: dict | None = None) -> str:
"""How a container reads to a person. A swarm task is its service and
slot: "codex_db · 1" for a replica, "agent · node-3" for a global
service's task on a node. Anything else as display_name reads it."""
from .swarm_view import _slot_key
svc = getattr(c, "service_name", None)
if not svc:
return display_name(c)
slot = _slot_key(c)
if slot.isdigit():
return f"{svc} · {slot}"
host = (node_hostname or {}).get(slot) or (node_hostname or {}).get(getattr(c, "node_id", None))
return f"{svc} · {host}" if host else svc
_CI_JOB = re.compile(
r"^(?:GITEA|FORGEJO)-ACTIONS-TASK-\d+-WORKFLOW-(?P<workflow>.+?)-JOB-(?P<job>.+?)(?:-[0-9a-f]{16,})?$")
# docker buildx's container driver names its node "buildx_buildkit_<builder><n>".
_BUILDKIT = re.compile(r"^buildx_buildkit_(?P<builder>.+?)\d*$")
def display_name(c) -> str:
"""A container's name as a list can show it in one short line (#5253): a
swarm task by its service (the host says where it runs), a Gitea or
Forgejo Actions job container as "CI: <workflow> · <job>", a buildx
builder as "BuildKit builder · <name>", anything else by its own name."""
svc = getattr(c, "service_name", None)
if svc:
return svc
m = _CI_JOB.match(c.name or "")
if m:
return f"CI: {m['workflow']} · {m['job']}"
m = _BUILDKIT.match(c.name or "")
if m:
return f"BuildKit builder · {m['builder']}"
return c.name
def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None, *,
thresholds: dict | None = None, norms: dict | None = None) -> dict:
"""A container's CPU and memory in terms a reader can weigh.
Docker's CPU% is per core (100% = one core busy), so it is shown as cores
and graded as a share of the host's CPUs when the host's core count is
known. Docker's memory limit is the host's RAM when none is set, so memory
is shown in bytes, against a limit only when one is actually set; mem_pct
is then a share of that limit, otherwise of the host's memory. `cpu_pct`
replaces the container's last sample (an average, say) when given.
With `thresholds`, cpu_grade and mem_grade are the grades to show: past
the cutoff but within the container's own normal for the hour (`norms`,
{metric: Norm}, container_norms) is "ok", and cpu_usual / mem_usual say
what is usual (#5742). Without, they are None and the page grades.
"""
cpu = getattr(c, "cpu_pct", None) if cpu_pct is None else cpu_pct
limit = getattr(c, "mem_limit_bytes", None)
limited = bool(limit and mem_total and limit < mem_total * 0.95)
out = {
"cores": None if cpu is None else cpu / 100.0,
"cpu_share": None if cpu is None or not cpus else cpu / cpus,
"cpus": cpus,
"mem_bytes": getattr(c, "mem_usage_bytes", None),
"mem_limit": limit if limited else None,
"mem_pct": getattr(c, "mem_pct", None),
"mem_of": "its limit" if limited else ("the host's memory" if mem_total else None),
"cpu_grade": None, "cpu_usual": None, "mem_grade": None, "mem_usual": None,
}
if thresholds is not None:
_grade(out, cpu, thresholds, norms or {})
return out
# Container figures graded against their own normal (#5742): the metric that
# learns it and its floor's setting (norms.floor.*).
NORMED = {"cpu": ("cpu_pct", "container_cpu_pct"), "mem": ("mem_pct", "container_mem_pct")}
def _grade(u: dict, cpu: float | None, thresholds: dict, norms: dict) -> None:
"""Fill usage()'s grades. CPU is graded as a share of the host but learned
per core (Docker's own figure), so its normal is read in cores."""
from steward.core.anomalies import departs, graded
from steward.core.formatting import status_class
from steward.core.settings import floor_of
grade = status_class(u["cpu_share"], "cpu", thresholds)
norm = norms.get("cpu_pct")
if grade in ("warn", "crit") and norm is not None and cpu is not None \
and not departs(cpu, norm, floor_of(thresholds, "container_cpu_pct")):
grade = "ok"
u["cpu_usual"] = f"usual {norm.usual_hold / 100:.2f} cores at this hour"
u["cpu_grade"] = grade
if u["mem_of"]:
u["mem_grade"], u["mem_usual"] = graded(
u["mem_pct"], "mem", thresholds, norms.get("mem_pct"),
floor_of(thresholds, "container_mem_pct"), "%")
else:
u["mem_grade"] = "dim"
async def container_norms(db, keys, now: datetime | None = None) -> dict:
"""{(host_id, container_name): {metric: Norm}}: each container's CPU and
memory normals for this hour. One without enough history is absent."""
from steward.core.norms import norms_at
resources = await container_resources(db, keys)
got = await norms_at(db, [("docker", r, metric) for r in resources
for metric, _f in NORMED.values()],
now or datetime.now(timezone.utc))
out: dict = {}
for (_s, resource, metric), norm in got.items():
out.setdefault(resources[resource], {})[metric] = norm
return out
async def host_capacity(db, host_ids) -> dict[str, tuple[int | None, int | None]]:
"""{host_id: (CPU cores, memory bytes)} from the host agent's latest
readings: cores counted from its per-core CPU series, memory from
available ÷ (1 − used%). (None, None) for a host without an agent."""
from sqlalchemy import or_
from steward.models.metrics import PluginMetricLatest
hosts = await host_map(db, set(host_ids))
if not hosts:
return {}
names = {h.name: hid for hid, h in hosts.items()}
rows = (await db.execute(
select(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name,
PluginMetricLatest.value)
.where(PluginMetricLatest.source_module == "host_agent",
or_(PluginMetricLatest.resource_name.in_(list(names)),
*(PluginMetricLatest.resource_name.like(escape_like(n) + ":core%", escape="\\")
for n in names)))
)).all()
cores: dict[str, int] = {}
mem: dict[str, dict] = {}
for res, metric, value in rows:
if res in names:
mem.setdefault(res, {})[metric] = value
elif metric == "cpu_pct":
host = res.rsplit(":core", 1)[0]
cores[host] = cores.get(host, 0) + 1
out = {}
for name, hid in names.items():
m = mem.get(name, {})
avail, used = m.get("mem_available_bytes"), m.get("mem_used_pct")
total = int(avail / (1 - used / 100)) if avail and used is not None and used < 99.5 else None
out[hid] = (cores.get(name), total)
return out
async def host_map(db, host_ids: set[str]) -> dict[str, Host]:
if not host_ids:
return {}
return {
h.id: h for h in (await db.execute(
select(Host).where(Host.id.in_(host_ids)))).scalars().all()
}
def escape_like(text: str) -> str:
"""Make `text` match literally inside a LIKE/ILIKE pattern (escape char \\)."""
return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
async def list_containers(db, host_id: str | None = None) -> list[DockerContainer]:
"""Containers, running first then by name; one host's when host_id is given."""
stmt = select(DockerContainer)
if host_id is not None:
stmt = stmt.where(DockerContainer.host_id == host_id)
stmt = stmt.order_by((DockerContainer.status == "running").desc(), DockerContainer.name)
return list((await db.execute(stmt)).scalars())
async def container_events(db, host_id: str, name: str, limit: int = 50) -> list[DockerEvent]:
"""A container's lifecycle events (start/stop/die/oom/health), newest first."""
return list((await db.execute(
select(DockerEvent)
.where(DockerEvent.host_id == host_id)
.where(DockerEvent.container_name == name)
.order_by(DockerEvent.at.desc())
.limit(limit)
)).scalars())
async def query_logs(db, host_id: str, name: str, stream: str, query: str,
limit: int) -> list:
"""The recent retained lines for one container, newest-first, optionally
filtered by stream and a case-insensitive substring.
Newest-first is deliberate: the viewer replaces the list on each poll, which
resets scroll to the top — so the latest lines stay visible without any
scroll handling, and older lines are a scroll away.
"""
stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line)
.where(DockerLog.host_id == host_id)
.where(DockerLog.container_name == name))
if stream in ("stdout", "stderr"):
stmt = stmt.where(DockerLog.stream == stream)
if query:
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\"))
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
return (await db.execute(stmt)).all()
CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes")
# A container's series live in the one metric store (#5752): source "docker",
# resource "<host name>/<container name>", recorded at ingest.
def container_resource(host_name: str, name: str) -> str:
return f"{host_name}/{name}"
async def container_resources(db, keys) -> dict[str, tuple[str, str]]:
"""{resource: (host_id, container_name)} for (host_id, container_name)
keys whose host still exists."""
keys = set(keys)
hosts = await host_map(db, {h for h, _ in keys})
return {container_resource(hosts[h].name, n): (h, n) for h, n in keys if h in hosts}
async def container_points(db, host_id: str, name: str, metric: str, since,
target: int = 120) -> list[tuple]:
"""(time, average) points of one container metric since `since`, oldest
first: raw samples bucket-averaged, and the hourly rollup for the part of
the range past the raw window (metric_series.points)."""
if metric not in CONTAINER_SERIES:
raise ValueError(f"metric must be one of {', '.join(CONTAINER_SERIES)}")
host = await db.get(Host, host_id)
if host is None:
return []
return await metric_series.points(db, "docker", container_resource(host.name, name),
metric, since, target=target)
async def _samples(db, resources, metrics, since: datetime, width: int | None = None):
"""Rows of (resource, time, metric, value) for the given containers since
`since`: each sample, or with `width` (seconds) each bucket's average."""
at = PluginMetric.recorded_at
if width:
at = func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, int(width)),
PluginMetric.recorded_at, func.to_timestamp(0))
value = func.avg(PluginMetric.value) if width else PluginMetric.value
q = (select(PluginMetric.resource_name, at.label("at"), PluginMetric.metric_name, value)
.where(PluginMetric.source_module == "docker",
PluginMetric.resource_name.in_(list(resources)),
PluginMetric.metric_name.in_(list(metrics)),
PluginMetric.recorded_at >= since))
if width:
q = q.group_by(PluginMetric.resource_name, at, PluginMetric.metric_name)
return (await db.execute(q.order_by(at))).all()
async def average_cpu(db, keys, since: datetime) -> dict[tuple[str, str], float]:
"""{(host_id, container_name): its average cpu_pct since `since`}."""
resources = await container_resources(db, keys)
if not resources:
return {}
rows = (await db.execute(
select(PluginMetric.resource_name, func.avg(PluginMetric.value))
.where(PluginMetric.source_module == "docker",
PluginMetric.resource_name.in_(list(resources)),
PluginMetric.metric_name == "cpu_pct",
PluginMetric.recorded_at >= since)
.group_by(PluginMetric.resource_name))).all()
return {resources[r]: float(v) for r, v in rows if v is not None}
# Busiest Containers bars (#5214): the recent past in BARS equal slices.
BARS = 24
def bar_slots(now: datetime, hours: int, bars: int = BARS) -> tuple[list[datetime], int]:
"""The start of each of the last `bars` buckets covering `hours`, oldest
first, aligned as date_bin aligns them (to the epoch), and their width in
seconds. The newest bucket is the one `now` falls in."""
width = max(60, hours * 3600 // bars)
last = int(now.timestamp()) // width * width
return [datetime.fromtimestamp(last - (bars - 1 - i) * width, timezone.utc)
for i in range(bars)], width
async def recent_usage(db, keys, since: datetime, width: int) -> dict:
"""{(host_id, container_name): {bucket start: (cpu_pct, mem_usage_bytes)}}
averaged per bucket since `since`, for the given containers, in one query."""
resources = await container_resources(db, keys)
if not resources:
return {}
out: dict = {}
for resource, at, metric, value in await _samples(
db, resources, ("cpu_pct", "mem_usage_bytes"), since, width):
cpu, mem = out.setdefault(resources[resource], {}).get(at, (None, None))
v = None if value is None else float(value)
out[resources[resource]][at] = (v, mem) if metric == "cpu_pct" else (cpu, v)
return out
def scaled_bars(values: list, floor: float) -> dict:
"""Bar heights in % of a roof: the row's own peak, but never below `floor`,
so a container idling at 0.05 cores draws flat and its 0.05 → 0.10 wobble
doesn't look like a 1.5 → 3 core spike. None (no samples) stays a gap."""
present = [v for v in values if v is not None]
peak = max(present) if present else None
roof = max(peak or 0.0, floor)
return {"bars": [None if v is None else round(100.0 * v / roof, 1) for v in values],
"peak": peak, "roof": roof}
async def search_logs(db, *, since, host_id: str | None = None, name: str | None = None,
containers: list[tuple[str, str]] | None = None,
text: str = "", stream: str = "", before=None, limit: int = 50) -> list:
"""Retained log lines matching a filter, newest first (limit + 1 fetched by callers
that need to know whether more exist). `containers` keeps the lines of those
(host_id, container_name) pairs: a swarm service's tasks (#5468)."""
from sqlalchemy import tuple_
stmt = (select(DockerLog.id, DockerLog.host_id, DockerLog.container_name,
DockerLog.ts, DockerLog.stream, DockerLog.line)
.where(DockerLog.ts >= since))
if host_id:
stmt = stmt.where(DockerLog.host_id == host_id)
if name:
stmt = stmt.where(DockerLog.container_name == name)
if containers is not None:
if not containers:
return []
stmt = stmt.where(tuple_(DockerLog.host_id, DockerLog.container_name).in_(containers))
if stream in ("stdout", "stderr"):
stmt = stmt.where(DockerLog.stream == stream)
if text:
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(text)}%", escape="\\"))
if before is not None:
stmt = stmt.where(DockerLog.ts < before)
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
return list((await db.execute(stmt)).all())
async def log_context(db, host_id: str, name: str, ts, n: int) -> tuple[list, list]:
"""Up to n lines either side of `ts` in one container's log, each oldest first."""
cols = (DockerLog.id, DockerLog.ts, DockerLog.stream, DockerLog.line)
base = select(*cols).where(DockerLog.host_id == host_id, DockerLog.container_name == name)
before = (await db.execute(
base.where(DockerLog.ts < ts).order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(n)
)).all()
after = (await db.execute(
base.where(DockerLog.ts > ts).order_by(DockerLog.ts, DockerLog.id).limit(n)
)).all()
return list(reversed(before)), list(after)
async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]:
"""{(host_id, container_name): [die/oom events since `since`, newest first]},
leaving out a die that was a stop (event_is_failure)."""
rows = (await db.execute(
select(DockerEvent)
.where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
.order_by(DockerEvent.at.desc())
)).scalars()
out: dict[tuple[str, str], list] = {}
for e in rows:
if event_is_failure(e):
out.setdefault((e.host_id, e.container_name), []).append(e)
return out
async def failure_events(db, since) -> list:
"""Die and OOM events since `since`, newest first; failures_24h() decides
which of them are failures."""
return list((await db.execute(
select(DockerEvent)
.where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
.order_by(DockerEvent.at.desc())
)).scalars())
async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None,
limit: int = 20) -> list[tuple[str, str, datetime]]:
"""(host_id, container_name, last seen) of containers Steward still holds
logs or events for, newest first — including ones Docker has removed.
exact: the name itself, from both logs and events. Otherwise names that
contain `want` (case-insensitive), from events only: a substring match
over every retained log line's name would scan the log table, and a
container Steward watched die has a die or stop event.
"""
seen: dict[tuple[str, str], datetime] = {}
sources = ((DockerLog, DockerLog.ts), (DockerEvent, DockerEvent.at)) if exact \
else ((DockerEvent, DockerEvent.at),)
for model, at in sources:
match = (model.container_name == want if exact
else model.container_name.ilike(f"%{escape_like(want)}%", escape="\\"))
stmt = (select(model.host_id, model.container_name, func.max(at))
.where(match).group_by(model.host_id, model.container_name))
if host_id:
stmt = stmt.where(model.host_id == host_id)
for hid, name, last in (await db.execute(stmt)).all():
key = (hid, name)
if key not in seen or last > seen[key]:
seen[key] = last
rows = sorted(((h, n, t) for (h, n), t in seen.items()), key=lambda r: r[2], reverse=True)
return rows[:limit]
# A swarm task's container is named "<service>.<slot>.<task id>": slot is the
# replica number, or the node id for a global service; task ids are 25 chars.
_TASK_SUFFIX = re.compile(r"^(?P<slot>[A-Za-z0-9]+)\.(?P<task>[a-z0-9]{25})$")
def task_of(service: str, container_name: str) -> tuple[str, str] | None:
"""(slot, task id) when `container_name` is one of `service`'s task
containers, else None — so "arr" does not claim "arr_radarr.1.…"."""
if not container_name.startswith(service + "."):
return None
m = _TASK_SUFFIX.match(container_name[len(service) + 1:])
return (m["slot"], m["task"]) if m else None
async def service_containers(db, service: str) -> list[tuple[str, str]]:
"""(host_id, container_name) of every task container of a swarm service
that Steward still holds anything for: the live ones, and the ones Docker
removed that left events (each task container has a start event). The
page's logs and history read across all of them (#5302)."""
pattern = escape_like(service + ".") + "%"
found: set[tuple[str, str]] = set()
for model, col in ((DockerContainer, DockerContainer.name),
(DockerEvent, DockerEvent.container_name)):
rows = (await db.execute(
select(model.host_id, col).where(col.like(pattern, escape="\\")).distinct())).all()
found.update((h, n) for h, n in rows if task_of(service, n))
return sorted(found)
async def query_service_logs(db, containers, stream: str, query: str, limit: int,
task: str | None = None) -> list:
"""The newest retained lines across a service's task containers, newest
first, each with its container name so the page can say which task wrote
it. `task` keeps one task's lines (a failure's "Logs →")."""
from sqlalchemy import tuple_
pairs = [(h, n) for h, n in containers if task is None or n.endswith("." + task)]
if not pairs:
return []
stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line, DockerLog.container_name)
.where(tuple_(DockerLog.host_id, DockerLog.container_name).in_(pairs)))
if stream in ("stdout", "stderr"):
stmt = stmt.where(DockerLog.stream == stream)
if query:
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\"))
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
return (await db.execute(stmt)).all()
def sum_series(samples, since: datetime, until: datetime, points: int = 60) -> list:
"""[(bucket start, cpu %, memory bytes)] summed across containers: each
container's samples are averaged within a bucket, then the containers are
added, so two replicas read as their combined load. `samples` are
(container_name, at, cpu_pct, mem_bytes); buckets with no sample are left
out rather than read as zero."""
width = max((until - since).total_seconds() / points, 60.0)
acc: dict[int, dict[str, list]] = {}
for name, at, cpu, mem in samples:
i = int((at - since).total_seconds() // width)
if i < 0:
continue
per = acc.setdefault(i, {}).setdefault(name, [0.0, 0, 0])
per[0] += cpu or 0.0
per[1] += mem or 0
per[2] += 1
out = []
for i in sorted(acc):
cpu = sum(c / n for c, _, n in acc[i].values())
mem = sum(m / n for _, m, n in acc[i].values())
out.append((since + timedelta(seconds=i * width), cpu, int(mem)))
return out
async def service_history(db, containers, since: datetime) -> list:
"""sum_series over a service's task containers' samples since `since`."""
resources = await container_resources(db, containers)
if not resources:
return []
joined: dict[tuple[str, datetime], list] = {}
for resource, at, metric, value in await _samples(
db, resources, ("cpu_pct", "mem_usage_bytes"), since):
pair = joined.setdefault((resource, at), [None, None])
pair[0 if metric == "cpu_pct" else 1] = value
rows = [(resource, at, cpu, mem) for (resource, at), (cpu, mem) in
sorted(joined.items(), key=lambda kv: kv[0][1])]
return sum_series(rows, since, datetime.now(timezone.utc))
async def unhealthy_since(db, keys) -> dict[tuple[str, str], datetime]:
"""{(host_id, container_name): when it last turned unhealthy} for the given
containers, from their health_change events; absent when none is retained."""
keys = set(keys)
if not keys:
return {}
rows = (await db.execute(
select(DockerEvent)
.where(DockerEvent.event == "health_change",
DockerEvent.detail.like("%→unhealthy"),
DockerEvent.container_name.in_({name for _, name in keys}))
.order_by(DockerEvent.at.desc())
)).scalars()
out: dict[tuple[str, str], datetime] = {}
for e in rows:
key = (e.host_id, e.container_name)
if key in keys and key not in out:
out[key] = e.at
return out
PRUNE_TARGETS = ("containers", "images", "system")
def prune_extra_vars(prune_target: str) -> dict:
"""Extra-vars for the bundled docker_prune playbook. "Prune unused images"
means ALL unused (docker image prune -a), not just dangling; the full system
prune stays conservative (-f, no -a) per the operator's choice (m78)."""
extra_vars: dict = {"prune_target": prune_target}
if prune_target == "images":
extra_vars["prune_all_images"] = True
return extra_vars
# ── what is wrong with a container ───────────────────────────────────────────
# Shared by the Docker page's "Needs attention" and the MCP tools, so both
# rank the same containers the same way.
PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
def container_problems(c, failures_1h: int, failed_24h: bool,
crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
"""What is wrong with one container, most severe first; [] when nothing is."""
status = (c.status or "").lower()
out = []
if status == "restarting":
out.append("restarting")
if failures_1h >= crash_loop_deaths:
out.append("crash_looping")
if status == "running" and (c.health or "") == "unhealthy":
out.append("unhealthy")
if c.oom_killed:
out.append("oom_killed")
if status in ("exited", "dead") and exit_is_failure(c.exit_code) and failed_24h:
out.append("exited_nonzero")
return out
def problem_since(problems: list[str], deaths: list, hour_ago: datetime,
unhealthy_at: datetime | None = None) -> datetime | None:
"""When a container's worst problem began; None when the retained events
cannot say. `deaths` are its die/oom events of the last 24h, newest first.
A restart or crash loop dates from its first death in the last hour, an
OOM kill or error exit from the death that ended it, and a failing
healthcheck from when its health last turned unhealthy.
"""
if not problems:
return None
worst = problems[0]
if worst in ("restarting", "crash_looping"):
in_hour = [e.at for e in deaths if e.at >= hour_ago]
return min(in_hour) if in_hour else (deaths[0].at if deaths else None)
if worst == "unhealthy":
return unhealthy_at
if worst == "oom_killed":
ooms = [e.at for e in deaths if e.event == "oom"]
return ooms[0] if ooms else (deaths[0].at if deaths else None)
return deaths[0].at if deaths else None
PROBLEM_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
"oom_killed": "warning", "exited_nonzero": "warning"}
PROBLEM_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
"unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
"exited_nonzero": "exited with an error"}
async def ranked_containers(db, now: datetime, crash_loop_deaths: int) -> list[dict]:
"""Every container (deduplicated) with what is wrong with it, most severe
first: {container, problems, since, replaced}. A swarm task Swarm has
replaced has no problems; its failure belongs to its service."""
containers = dedup_by_container_id(await list_containers(db))
services = list((await db.execute(select(DockerSwarmService))).scalars())
replaced = replaced_tasks(containers, services)
failures = await recent_failures_by_container(db, now - timedelta(hours=24))
hour_ago = now - timedelta(hours=1)
sick = await unhealthy_since(
db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"])
out = []
for c in containers:
key = (c.host_id, c.name)
gone = key in replaced
events = failures.get(key, [])
recent = sum(1 for e in events if e.at >= hour_ago)
problems = [] if gone else container_problems(c, recent, bool(events), crash_loop_deaths)
out.append({"container": c, "problems": problems, "replaced": gone,
"since": problem_since(problems, events, hour_ago, sick.get(key))})
return out
def needs_attention(ranked: list[dict]) -> list[dict]:
"""The containers with something wrong, worst problem first, then by host
and name."""
return sorted((r for r in ranked if r["problems"]),
key=lambda r: (PROBLEMS.index(r["problems"][0]), r["container"].host_id, r["container"].name))
def stale_after_seconds(config) -> int:
"""How old a container's last report may be before the page calls it
stale: the host agent's own staleness window, since the agent reports it."""
return int(((config.get("PLUGINS") or {}).get("host_agent") or {}).get("stale_after_seconds", 180))
async def disk_hosts(db, since: datetime, *, points: int = 60) -> list[dict]:
"""Each Docker host's disk as `docker system df` last said, most space
first, with its recorded total since `since` (#5304): the points, oldest
first, and the growth over them (disk_view.growth; None until there are
two). Hosts are named by their Steward host."""
from steward.core import metric_series
from .disk_view import DISK_PARTS, growth
from .models import DockerDiskUsage
summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
hosts = await host_map(db, {s.host_id for s in summaries})
out = []
for s in summaries:
name = hosts[s.host_id].name if s.host_id in hosts else s.host_id
parts = {m: int(getattr(s, f) or 0) for m, f in DISK_PARTS.items()}
series = await metric_series.points(db, "docker", name, "disk_total_bytes", since, target=points)
out.append({"host_id": s.host_id, "host_name": name, "as_of": s.updated_at,
"total": sum(parts.values()), "parts": parts,
"reclaimable": int(s.images_reclaimable or 0), "points": series,
"growth": growth(series)})
return sorted(out, key=lambda h: (-h["total"], h["host_name"].lower()))
plugins/docker/swarm_nodes.py
-71
View File
@@ -1,71 +0,0 @@
# plugins/docker/swarm_nodes.py
"""The Swarm page laid out by node (#5303): one card per node with its role,
whether it takes work, what its containers use against what the machine has,
and the services Swarm placed on it. Model-free, so it is unit-tested without
a database; routes.swarm() feeds it.
A node with a Steward agent reports its own containers, so its services carry
CPU and memory. A node without one is known only from the managers' placement:
its services are listed by count, with no figures.
"""
from __future__ import annotations
from .swarm_view import _placement
def node_ok(n) -> bool:
"""A node that is up and taking work."""
return n.status == "ready" and n.availability == "active"
def node_cards(nodes, services, containers, *, node_host: dict, host_names: dict,
capacity: dict, service_state: dict) -> list[dict]:
"""One card per node, a node in trouble first, then managers, then by name.
`nodes` and `services` are one swarm's deduplicated rows; `containers`
every container row (only running swarm tasks on these nodes count);
`node_host` maps a node id to the Steward host on it, `capacity` a host id
to (cores, memory bytes), `service_state` a service name to healthy,
degraded or down."""
running_on: dict[str, dict[str, dict]] = {}
for c in containers:
if not (c.node_id and c.service_name and c.status == "running"):
continue
row = running_on.setdefault(c.node_id, {}).setdefault(
c.service_name, {"name": c.service_name, "tasks": 0, "cpu_pct": 0.0, "mem_bytes": 0,
"measured": True})
row["tasks"] += 1
row["cpu_pct"] += c.cpu_pct or 0.0
row["mem_bytes"] += c.mem_usage_bytes or 0
placed: dict[str, dict[str, int]] = {}
for s in services:
for p in _placement(s):
count = int(p.get("running") or 0)
if count > 0 and p.get("node_id"):
placed.setdefault(p["node_id"], {})[s.service_name] = count
cards = []
for n in nodes:
rows = dict(running_on.get(n.node_id, {}))
for name, count in placed.get(n.node_id, {}).items():
rows.setdefault(name, {"name": name, "tasks": count, "cpu_pct": None, "mem_bytes": None,
"measured": False})
for r in rows.values():
r["state"] = service_state.get(r["name"], "healthy")
r["cores"] = r["cpu_pct"] / 100 if r["cpu_pct"] is not None else None
tasks = sorted(rows.values(), key=lambda r: (r["state"] == "healthy", -(r["cores"] or 0.0),
r["name"].lower()))
host_id = node_host.get(n.node_id)
cores, mem = capacity.get(host_id, (None, None)) if host_id else (None, None)
measured = [r for r in tasks if r["measured"]]
cards.append({
"node_id": n.node_id, "hostname": n.hostname or n.node_id[:12], "role": n.role,
"leader": n.leader, "status": n.status, "availability": n.availability, "ok": node_ok(n),
"host_id": host_id, "host_name": host_names.get(host_id) if host_id else None,
"agent": bool(running_on.get(n.node_id)) or host_id is not None,
"tasks": tasks, "task_count": sum(r["tasks"] for r in tasks),
"cores_used": sum(r["cores"] for r in measured) if measured else None,
"mem_used": sum(r["mem_bytes"] for r in measured) if measured else None,
"cores": cores, "mem": mem,
})
return sorted(cards, key=lambda c: (c["ok"], c["role"] != "manager", c["hostname"].lower()))
plugins/docker/swarm_view.py
+14 -262
View File
@@ -11,28 +11,12 @@ For every swarm service we list the real replicas we have detail for, then add
"ghost" replicas for the remaining placement count on each node (agent-less
nodes). Non-swarm containers pass through grouped by host, unchanged.
A third source, DockerSwarmService.tasks_json (agent ≥ 1.8.0), is Swarm's own
record of each service's newest tasks. It decides which task containers are
*superseded* — replaced by Swarm, so neither a replica nor a crash — and gives
each service its failures across task retries and the error of its last
failed task. Swarm restarts a task by creating a new one (a new container), so
failures can only be counted per service, never per container.
Kept model-free (no ORM imports) so it's unit-testable and safe to import in
tests without the plugin-loader "table already defined" gotcha.
"""
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
# Task states that mean the task ended in error: `failed` (its container exited
# non-zero or never started) and `rejected` (Swarm could not start it at all,
# e.g. no such image, a bad mount).
FAILED_TASK_STATES = ("failed", "rejected")
# Desired states of a task Swarm still means to run. Anything else (shutdown,
# remove, complete, orphaned) is a task Swarm has finished with.
_CURRENT_DESIRED = ("running", "ready", "")
def _service_state(running: int, desired: int) -> str:
@@ -52,222 +36,30 @@ def _placement(service) -> list[dict]:
return [p for p in data if isinstance(p, dict)]
def parse_tasks(service) -> list[dict]:
"""A service row's task records, newest first; [] when its agent sent none."""
raw = getattr(service, "tasks_json", None) or "[]"
try:
data = json.loads(raw)
except (ValueError, TypeError):
return []
return [t for t in data if isinstance(t, dict)]
def build_swarm_services(containers, services, node_hostname: dict, host_name: dict) -> dict:
"""Return {"services": [...], "standalone": [...]}.
def task_time(task: dict) -> datetime | None:
"""A task's status timestamp. Docker writes nanoseconds; Python ≥ 3.11 keeps
the first six digits."""
raw = task.get("at") or ""
try:
at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
except ValueError:
return None
return at if at.tzinfo else at.replace(tzinfo=timezone.utc)
def task_failures(tasks: list[dict], since: datetime) -> int:
"""How many tasks ended in error at or after `since`."""
n = 0
for t in tasks:
at = task_time(t)
if t.get("state") in FAILED_TASK_STATES and at is not None and at >= since:
n += 1
return n
def last_task_error(tasks: list[dict]) -> dict | None:
"""Why the service's newest troubled task is in trouble, from Swarm itself.
The newest task that failed or was rejected, or that Swarm attached an error
to while still trying (a pending task with "no suitable node", say). None
when no retained task has a problem.
"""
for t in tasks: # newest first
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
return {"state": t.get("state"), "error": t.get("error") or None,
"message": t.get("message") or None, "exit_code": t.get("exit_code"),
"node_id": t.get("node_id") or None, "at": t.get("at") or None,
"task_id": t.get("task_id") or None, "slot": t.get("slot")}
return None
def task_summary(tasks: list[dict], now: datetime, crash_loop_deaths: int,
node_hostname: dict) -> dict:
"""What a service's task records say: tasks_known, failures_1h,
failures_24h (None without records), crash_looping, and last_error with
the node's hostname as `node`."""
known = bool(tasks)
failures_1h = task_failures(tasks, now - timedelta(hours=1)) if known else None
last_error = last_task_error(tasks)
if last_error is not None:
nid = last_error["node_id"]
last_error["node"] = node_hostname.get(nid) or (nid[:12] if nid else None)
at = task_time(last_error)
last_error["age_s"] = max(0, int((now - at).total_seconds())) if at else None
return {
"tasks_known": known,
"failures_1h": failures_1h,
"failures_24h": task_failures(tasks, now - timedelta(hours=24)) if known else None,
"crash_looping": bool(known and failures_1h >= crash_loop_deaths),
"last_error": last_error,
}
def _oldest_failure(tasks: list[dict], since: datetime) -> datetime | None:
times = [at for t in tasks if t.get("state") in FAILED_TASK_STATES
and (at := task_time(t)) is not None and at >= since]
return min(times, default=None)
def trouble_since(tasks: list[dict], state: str, crash_looping: bool,
now: datetime) -> datetime | None:
"""When a service's current trouble began, as far as its retained tasks
tell; None when it has none, or the tasks cannot say.
A crash loop dates from its first failure in the last hour, and a healthy
service that lost tasks from its first failure in the last 24h. A service
short of replicas dates from its troubled slots (replicated: the slot;
global: the node): in each slot whose newest task is not running, the
oldest of its newest troubled tasks (failed, rejected, or carrying an
error), walking back until a running task interrupts them. Swarm keeps few
old tasks, so a long outage reads as the oldest one retained: a lower
bound.
"""
if crash_looping:
return _oldest_failure(tasks, now - timedelta(hours=1))
if state == "healthy":
return _oldest_failure(tasks, now - timedelta(hours=24))
by_slot: dict = {}
for t in tasks: # newest first, so each slot's list is too
by_slot.setdefault(t.get("slot") or t.get("node_id"), []).append(t)
began = []
for slot_tasks in by_slot.values():
oldest = None
for t in slot_tasks:
if t.get("state") == "running":
break
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
oldest = task_time(t) or oldest
if oldest is not None:
began.append(oldest)
return min(began, default=None)
def _slot_key(c) -> str:
"""A task container's slot within its service: "<service>.<slot>.<task>" for
replicated services, "<service>.<node id>.<task>" for global ones."""
svc, name = c.service_name, c.name or ""
rest = name[len(svc) + 1:] if name.startswith(svc + ".") else name
return rest.split(".", 1)[0]
def superseded_containers(swarm_cs, tasks_by_id: dict[str, dict]) -> set[tuple]:
"""(host_id, name) of task containers Swarm has replaced.
With the task on record, its desired state decides: Swarm sets a replaced
or failed task's desired state to shutdown. Without it (an older agent, or a
task older than the retained history), a container that is not running is
superseded when another task in the same slot is running or started later.
"""
out: set[tuple] = set()
by_slot: dict[tuple, list] = {}
for c in swarm_cs:
by_slot.setdefault((c.service_name, _slot_key(c)), []).append(c)
for c in swarm_cs:
task = tasks_by_id.get(getattr(c, "task_id", None) or "")
if task is not None:
if task.get("desired_state") not in _CURRENT_DESIRED:
out.add((c.host_id, c.name))
continue
if (getattr(c, "status", "") or "").lower() == "running":
continue
mine = getattr(c, "started_at", None)
for other in by_slot[(c.service_name, _slot_key(c))]:
if other is c:
continue
theirs = getattr(other, "started_at", None)
if ((getattr(other, "status", "") or "").lower() == "running"
or (theirs is not None and (mine is None or theirs > mine))):
out.add((c.host_id, c.name))
break
return out
def _newest_per_name(services) -> dict[str, object]:
"""Every manager reports the same cluster-global service set; keep the most
recently updated row per name, so a manager on an older agent (no tasks)
doesn't shadow one that reports them."""
by_name: dict[str, object] = {}
for s in services:
name = getattr(s, "service_name", None)
if not name:
continue
kept = by_name.get(name)
if kept is None:
by_name[name] = s
continue
a, b = getattr(s, "updated_at", None), getattr(kept, "updated_at", None)
if a is not None and (b is None or a > b):
by_name[name] = s
return by_name
def replaced_tasks(containers, services) -> set[tuple]:
"""(host_id, name) of the task containers Swarm has replaced, from the
container rows and the managers' service rows. What the container views use
to leave history out of their problem lists."""
swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
tasks_by_id = {t["task_id"]: t for s in _newest_per_name(services).values()
for t in parse_tasks(s) if t.get("task_id")}
return superseded_containers(swarm_cs, tasks_by_id)
def build_swarm_services(containers, services, node_hostname: dict, host_name: dict,
*, now: datetime | None = None, crash_loop_deaths: int = 3) -> dict:
"""Return {"services": [...], "standalone": [...], "superseded": {(host_id, name)}}.
services[i] = {name, mode, image, desired, running, state, replicas:[...],
previous_tasks, tasks_known, failures_1h, failures_24h,
crash_looping, last_error}
services[i] = {name, mode, desired, running, state, replicas:[...]}
real replica = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
mem_bytes, started_at, image, health, restart_count}
# current tasks only
health, restart_count}
ghost replica = {ghost:True, host, count} # placement with no local row
previous_tasks = task containers Swarm replaced, left out of replicas
failures_* = tasks that failed or were rejected in the window (None
without task data); crash_looping = failures_1h ≥ the
Settings → Thresholds crash-loop count
last_error = last_task_error(), plus `node` (the node's hostname)
trouble_since = trouble_since(): when the current trouble began, or None
standalone[i] = {host_id, host, containers:[...]} # non-swarm, by host
superseded = the task containers Swarm replaced, so callers can skip them
"""
now = now or datetime.now(timezone.utc)
containers = list(containers)
swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
standalone_cs = [c for c in containers if not getattr(c, "service_name", None)]
svc_by_name = _newest_per_name(services)
tasks_by_svc = {name: parse_tasks(s) for name, s in svc_by_name.items()}
tasks_by_id = {t["task_id"]: t for ts in tasks_by_svc.values() for t in ts
if t.get("task_id")}
superseded = superseded_containers(swarm_cs, tasks_by_id)
# Dedup service rows by name (every manager reports the same cluster-global
# set); keep the first — placement is identical across managers.
svc_by_name: dict[str, object] = {}
for s in services:
name = getattr(s, "service_name", None)
if name and name not in svc_by_name:
svc_by_name[name] = s
# Current task containers grouped by service name; replaced ones are counted.
# Real containers grouped by service name.
cs_by_service: dict[str, list] = {}
previous: dict[str, int] = {}
for c in swarm_cs:
if (c.host_id, c.name) in superseded:
previous[c.service_name] = previous.get(c.service_name, 0) + 1
continue
cs_by_service.setdefault(c.service_name, []).append(c)
def _host_of(c):
@@ -290,9 +82,6 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
"status": getattr(c, "status", "unknown"),
"cpu_pct": getattr(c, "cpu_pct", None),
"mem_pct": getattr(c, "mem_pct", None),
"mem_bytes": getattr(c, "mem_usage_bytes", None),
"started_at": getattr(c, "started_at", None),
"image": getattr(c, "image", None) or None,
"health": getattr(c, "health", None),
"restart_count": getattr(c, "restart_count", 0) or 0,
})
@@ -328,20 +117,13 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
if desired is None:
desired = len(replicas)
tasks = tasks_by_svc.get(name, [])
state = _service_state(int(running or 0), int(desired or 0))
summary = task_summary(tasks, now, crash_loop_deaths, node_hostname)
out_services.append({
"name": name,
"mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
"image": getattr(svc, "image", None) if svc is not None else None,
"running": running,
"desired": desired,
"state": state,
"state": _service_state(int(running or 0), int(desired or 0)),
"replicas": replicas,
"previous_tasks": previous.get(name, 0),
**summary,
"trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
})
# Non-swarm containers grouped by host, running-first order preserved.
@@ -355,34 +137,4 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
g["containers"].append(c)
standalone = sorted(groups.values(), key=lambda g: g["host"].lower())
return {"services": out_services, "standalone": standalone, "superseded": superseded}
def service_trouble(s: dict) -> int | None:
"""How bad a built service is, for sorting and for Needs attention: 0 a
crash loop, 1 no replica running, 2 short of replicas, 3 healthy but with
failed tasks in the last 24h; None when nothing is wrong."""
if s.get("crash_looping"):
return 0
if s.get("state") == "down":
return 1
if s.get("state") == "degraded":
return 2
if s.get("failures_24h"):
return 3
return None
def services_problems_first(services: list[dict]) -> list[dict]:
"""Services in trouble first, worst first, then the rest by name."""
return sorted(services, key=lambda s: (
4 if service_trouble(s) is None else service_trouble(s), s["name"].lower()))
def failing_services(services: list[dict]) -> list[dict]:
"""The services that belong in Needs attention: a crash loop, or short of
their desired replicas. A healthy service whose failed tasks Swarm has
already replaced is history, shown on the service, not a problem now."""
# A crash loop ranks 0, so test the rank against None, not for truth.
return [s for s in services_problems_first(services)
if (t := service_trouble(s)) is not None and t <= 2]
return {"services": out_services, "standalone": standalone}
plugins/docker/templates/docker/_container_history.html
+17
View File
@@ -0,0 +1,17 @@
{# docker/_container_history.html — CPU/mem sparklines for the selected range #}
{% if have_data %}
<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:1.5rem;">
<div>
<div class="section-title" style="margin-bottom:0.4rem;">CPU %</div>
{{ sparkline_cpu | safe }}
</div>
<div>
<div class="section-title" style="margin-bottom:0.4rem;">Memory %</div>
{{ sparkline_mem | safe }}
</div>
</div>
{% else %}
<div style="color:var(--text-muted);font-size:0.85rem;">
Not enough samples in this range yet — history fills in as the agent reports.
</div>
{% endif %}
plugins/docker/templates/docker/_container_logs_lines.html
-24
View File
@@ -1,24 +0,0 @@
{# docker/_container_logs_lines.html — log-line fragment, HTMX-polled (m79).
{{ l.line }} is auto-escaped by Jinja, so log content can't inject markup.
A service's lines (#5302) carry the task each came from. #}
{% if lines %}
{% for l in lines %}
<div style="display:flex;gap:0.6rem;white-space:pre-wrap;word-break:break-word;">
<span style="color:var(--text-dim);flex-shrink:0;" title="{{ l.ts }}">{{ l.ts.strftime("%m-%d %H:%M:%S") }}</span>
{% if l.task %}<span class="tone-muted" style="flex-shrink:0;white-space:nowrap;" title="Slot and task id">{{ l.task }}</span>{% endif %}
<span style="flex-shrink:0;width:3.2rem;color:{% if l.stream == 'stderr' %}var(--red){% else %}var(--text-muted){% endif %};">{{ l.stream }}</span>
<span style="flex:1;min-width:0;{% if l.stream == 'stderr' %}color:var(--text);{% endif %}">{{ l.line }}</span>
</div>
{% endfor %}
{% else %}
<div style="color:var(--text-muted);padding:1rem 0.25rem;">
{% if filtered %}
No lines match the current filter.
{% else %}
No logs collected yet for <code>{{ name }}</code>. Lines appear within a few
seconds of {{ "one of its tasks" if service else "the container" }} writing to stdout/stderr — unless it's on the log
exclude list or log collection is turned off in
<a href="/settings/thresholds/">Settings → Thresholds &amp; Retention</a>.
{% endif %}
</div>
{% endif %}
plugins/docker/templates/docker/_disk_hosts.html
-37
View File
@@ -1,37 +0,0 @@
{# docker/_disk_hosts.html — every Docker host's disk side by side (#5304):
what it uses now, how much that grew over the selected range, and a line
of it; the parts after. A host is linked to its own section below. #}
{% if rows %}
<table class="table">
<thead><tr>
<th>Host</th>
<th class="num">Docker uses</th>
<th class="num" title="Change in the total over the last {{ range_key }}, from Steward's own record of it">Change ({{ range_key }})</th>
<th>Over {{ range_key }}</th>
<th class="num">Images</th><th class="num">Volumes</th><th class="num" title="Written by containers on top of their images">Container files</th>
<th class="num">Build cache</th><th class="num" title="Held by images no container uses">Reclaimable</th>
</tr></thead>
<tbody>
{% for r in rows %}
{% set g = r.growth %}
<tr>
<td><a href="#host-{{ r.host_id }}">{{ r.host_name }}</a></td>
<td class="num"><strong>{{ r.total|bytes }}</strong></td>
<td class="num">
{%- if g is none %}<span class="tone-dim" title="Steward records this from now on; growth shows once there are two readings">—</span>
{%- else %}<span class="tone-{{ 'warn' if g.pct is not none and g.pct >= 10 else ('ok' if g.change < 0 else 'muted') }}"
title="{{ g.from|bytes }} → {{ g.to|bytes }} since {{ g.since.strftime('%Y-%m-%d %H:%M UTC') }}">{{ "+" if g.change > 0 else ("−" if g.change < 0 else "±") }}{{ (g.change|abs)|bytes }}</span>{% endif -%}
</td>
<td class="tone-info">{{ r.spark|safe }}</td>
<td class="num">{{ r.parts.disk_images_bytes|bytes }}</td>
<td class="num">{{ r.parts.disk_volumes_bytes|bytes }}</td>
<td class="num">{{ r.parts.disk_containers_bytes|bytes }}</td>
<td class="num">{{ r.parts.disk_build_cache_bytes|bytes }}</td>
<td class="num">{% if r.reclaimable %}<span class="tone-warn">{{ r.reclaimable|bytes }}</span>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p class="tone-muted" style="margin:0;padding:1rem;">No host has reported its Docker disk yet.</p>
{% endif %}
plugins/docker/templates/docker/_swarm_failures.html
-25
View File
@@ -1,25 +0,0 @@
{#- One line on a swarm service's task failures, from Swarm's own task records
(agent ≥ 1.8.0): a crash loop or the failures of the last 24h, and how the
newest troubled task ended, in Swarm's words. Expects `s` with state,
crash_looping, failures_1h, failures_24h and last_error; `style` is
optional extra CSS for the wrapper. Renders nothing for a healthy service
with no recent failures. -#}
{%- set e = s.last_error -%}
{%- if s.crash_looping or s.failures_24h or (e and s.state != 'healthy') %}
<div style="display:flex;flex-wrap:wrap;align-items:baseline;gap:0.2rem 0.6rem;font-size:0.78rem;color:var(--text-muted);{{ style or '' }}">
{% if s.crash_looping %}
<span style="color:var(--red);font-weight:500;">Crash-looping · {{ s.failures_1h }} tasks failed in the last hour</span>
{% elif s.failures_24h %}
<span style="color:var(--orange);">{{ s.failures_24h }} task{{ 's' if s.failures_24h != 1 }} failed in the last 24h</span>
{% endif %}
{% if e %}
<span title="How Swarm recorded the end of this service's newest failed task">
Last task {{ e.state }}:
<span style="color:var(--text);font-family:ui-monospace,monospace;overflow-wrap:anywhere;">{{ e.error or e.message or e.state }}</span>
{%- if e.exit_code %} · exit {{ e.exit_code }}{% endif %}
{%- if e.node %} · {{ e.node }}{% endif %}
{%- if e.age_s is not none %} · {{ e.age_s|duration }} ago{% endif %}
</span>
{% endif %}
</div>
{%- endif %}
plugins/docker/templates/docker/_usage.html
-28
View File
@@ -1,28 +0,0 @@
{#- A container's CPU and memory as service.usage() describes them (#5209).
Docker counts CPU per core (100% = one core busy), so CPU reads as cores,
toned by its share of the host's CPUs when the host's core count is known.
Memory reads in bytes, against a limit only when one is actually set;
mem_pct is then a share of that limit, otherwise of the host's memory.
Each is toned by usage()'s grade when it has one: within the container's
own normal for the hour reads healthy, and says what is usual (#5742). -#}
{% macro cores(n) -%}
{{ "%.2f"|format(n) if n < 1 else "%.1f"|format(n) }} {{ "core" if n == 1 else "cores" }}
{%- endmacro %}
{% macro cpu(u) -%}
{%- if u.cores is none -%}<span class="tone-dim">—</span>
{%- else -%}
<span class="tone-{{ u.cpu_grade or status_class(u.cpu_share, 'cpu') }}"
title="Docker counts CPU per core: {{ '%.0f'|format(u.cores * 100) }}% of one core{% if u.cpus %}, {{ '%.1f'|format(u.cpu_share) }}% of this host's {{ u.cpus }} core{{ 's' if u.cpus != 1 }}{% endif %}{% if u.cpu_usual %}; {{ u.cpu_usual }}{% endif %}">{{ cores(u.cores) }}</span>
{%- endif -%}
{%- endmacro %}
{% macro mem(u) -%}
{%- if u.mem_bytes is none -%}<span class="tone-dim">—</span>
{%- else -%}
<span class="tone-{{ u.mem_grade or (status_class(u.mem_pct, 'mem') if u.mem_of else 'dim') }}"
title="{% if u.mem_pct is not none and u.mem_of %}{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% if u.mem_usual %}, {{ u.mem_usual }}{% endif %}{% else %}No memory limit set{% endif %}">{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %}</span>
{%- endif -%}
{%- endmacro %}
plugins/docker/templates/docker/_usage_history.html
-23
View File
@@ -1,23 +0,0 @@
{# docker/_usage_history.html — CPU and memory over the selected range, for a
service (summed across its replicas, #5302) or one container (#5310): the
latest and the peak, over a line that spans its half. The line is drawn
against at least cpu_floor / mem_floor, so idling near zero draws flat. #}
{% from "docker/_usage.html" import cores %}
{% if have_data %}
<div class="history-pair">
<div>
<div class="section-title">CPU</div>
<div style="font-variant-numeric:tabular-nums;">{{ cores(cpu_now) }} <span class="tone-dim">now · peak {{ cores(cpu_peak) }}</span></div>
<div class="spark" title="Drawn from zero to its peak, or to {{ cores(cpu_floor)|striptags }} when the peak is lower">{{ spark_cpu|safe }}</div>
</div>
<div>
<div class="section-title">Memory</div>
<div style="font-variant-numeric:tabular-nums;">{{ mem_now|bytes }} <span class="tone-dim">now · peak {{ mem_peak|bytes }}</span></div>
<div class="spark" title="Drawn from zero to its peak, or to {{ mem_floor|bytes }} when the peak is lower">{{ spark_mem|safe }}</div>
</div>
</div>
{% else %}
<p class="tone-muted" style="margin:0;font-size:0.85rem;">
Not enough samples in the last {{ range_key }} yet. History fills in as the agent reports.
</p>
{% endif %}
plugins/docker/templates/docker/container_detail.html
+78 -97
View File
@@ -1,98 +1,78 @@
{# docker/container_detail.html — one container (#5310), shaped like a
service's page: a header that says whether it is well and what it is; why
it stopped, only while it has; what it moved since it started; its CPU and
memory over time (the one home for what it uses now), with its metric
anomalies above when it has any; its lifecycle.
A swarm task is titled by its service and slot, the raw name under it. #}
{# docker/container_detail.html — full detail page for one container #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, status_badge %}
{% block title %}{{ label or name }} — Docker — Steward{% endblock %}
{% block breadcrumb %}
{%- if container is not none and container.service_name -%}
{{ crumbs([("Docker", "/plugins/docker/"), (container.service_name, "/plugins/docker/service/" ~ container.service_name), (label, "")]) }}
{%- else -%}
{{ crumbs([("Docker", "/plugins/docker/"), (label or name, "")]) }}
{%- endif -%}
{% endblock %}
{% from "_macros.html" import crumbs %}
{% block title %}{{ name }} — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
{% block content %}
{% if container is none %}
<div class="card empty-state">
<strong>Container not found</strong>
<span>No container named <code>{{ name }}</code> is reported for this host now. It may have been
removed, or the host agent has not reported recently. <a href="/plugins/docker/">Back to Docker</a></span>
<div class="card" style="text-align:center;padding:3rem;">
<div style="font-weight:600;margin-bottom:0.4rem;">Container not found</div>
<div style="color:var(--text-muted);font-size:0.9rem;">
No container named <code>{{ name }}</code> is currently reported for this host.
It may have been removed, or the host agent hasn't reported recently.
</div>
</div>
{% else %}
{% set c = container %}
{# ── Header: the verdict, then what it is ────────────────────────────────── #}
<div class="page-header">
<div style="min-width:0;">
<h1 class="page-title" style="overflow-wrap:anywhere;">{{ label }}</h1>
<p class="page-subtitle">
{% if c.status == 'running' %}{{ status_badge('ok', 'Running') }}
{% elif c.status == 'restarting' %}{{ status_badge('crit', 'Restarting') }}
{% elif c.status == 'paused' %}{{ status_badge('warn', 'Paused') }}
{% elif exit_failed %}{{ status_badge('crit', c.status|capitalize) }}
{% else %}{{ status_badge('dim', c.status|capitalize) }}{% endif %}
{%- if c.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}
{%- elif c.health == 'starting' %} {{ status_badge('pending', 'Starting') }}
{%- elif c.health == 'healthy' %} {{ status_badge('ok', 'Healthy') }}{% endif %}
{%- if uptime %} · up {{ uptime }}{% endif %}
{%- if host %} · on <a href="/hosts/{{ host.id }}">{{ host.name }}</a>{% endif %}
{%- if c.service_name %} · task of <a href="/plugins/docker/service/{{ c.service_name }}">{{ c.service_name }}</a>
{%- elif c.compose_project %} · compose <span class="chip">{{ c.compose_project }}</span>{% endif %}
{%- if c.image %} · <span class="mono" style="overflow-wrap:anywhere;" title="{{ c.image }}">{{ c.image.split("@")[0] }}</span>{% endif %}
</p>
<p class="page-subtitle tone-dim" style="margin-top:0.2rem;">
{%- if label != c.name %}<span class="mono" style="overflow-wrap:anywhere;">{{ c.name }}</span> · {% endif -%}
{%- if c.scraped_at %}reported {{ c.scraped_at|ago }}{% endif -%}
</p>
</div>
<div class="page-actions"><a class="btn btn-sm btn-ghost" href="/plugins/docker/container/{{ host_id }}/{{ name }}/logs">Logs</a></div>
{# ── Header ──────────────────────────────────────────────────────────────── #}
<div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
<span class="dot {% if container.status == 'running' %}dot-up{% elif container.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<h1 class="page-title" style="margin-bottom:0;">{{ container.name }}</h1>
{% if container.health == 'healthy' %}<span style="font-size:0.72rem;color:var(--green);border:1px solid var(--green);border-radius:3px;padding:0.05rem 0.4rem;">healthy</span>
{% elif container.health == 'unhealthy' %}<span style="font-size:0.72rem;color:var(--red);border:1px solid var(--red);border-radius:3px;padding:0.05rem 0.4rem;">unhealthy</span>
{% elif container.health == 'starting' %}<span style="font-size:0.72rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0.05rem 0.4rem;">starting</span>{% endif %}
</div>
<div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
{{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %}
{% if host %} · on <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %}
</div>
{# ── Why it stopped, only while it has ───────────────────────────────────── #}
{% if c.status != 'running' and exit_failed %}
<section class="card problem-card" aria-labelledby="ctr-exit">
<h2 id="ctr-exit" class="section-title" style="margin-bottom:0.4rem;">Last exit</h2>
<p style="margin:0;">
Exited <span class="tone-crit">{{ c.exit_code }}</span>{% if c.oom_killed %}: killed for running out of memory{% endif %}
{%- if died_at %} · {{ died_at|ago }}{% endif %}
· <a href="/plugins/docker/container/{{ host_id }}/{{ name }}/logs">its logs →</a>
</p>
</section>
{% endif %}
{# ── Facts grid ──────────────────────────────────────────────────────────── #}
{% macro fact(label, value, colour="") %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">{{ label }}</div>
<div style="font-size:0.95rem;{% if colour %}color:{{ colour }};{% endif %}font-family:ui-monospace,monospace;">{{ value }}</div>
</div>
{% endmacro %}
{# ── Its CPU or memory out of its normal for the hour, and past anomalies (#5742) #}
{% with back=anomaly_back, base_resource=anomaly_resource %}{% include "_anomaly_card.html" %}{% endwith %}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
{{ fact("CPU", "%.1f%%" | format(container.cpu_pct) if container.cpu_pct is not none else "—") }}
{{ fact("Memory", "%.1f%%" | format(container.mem_pct) if container.mem_pct is not none else "—") }}
{{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }}
{{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—",
"var(--red)" if (container.exit_code is not none and container.exit_code != 0) else "") }}
{{ fact("Net in", net_rx) }}
{{ fact("Net out", net_tx) }}
{{ fact("Block read", blk_read) }}
{{ fact("Block write", blk_write) }}
</div>
{# ── What it is set up with, and what it moved since it started ─────────── #}
<div class="card">
<dl class="kv">
{% if not c.service_name %}
{#- Swarm never restarts a task's container: it replaces the task, so a
task's restart count is always 0; its service's Tasks table is the
count that means something. #}
<dt>Restarts</dt><dd>{% if c.restart_count %}<span class="tone-warn">{{ c.restart_count }}</span>{% else %}0{% endif %}
{%- if c.exit_code is not none and c.status == 'running' and c.restart_count %} <span class="tone-dim">· last exit {{ c.exit_code }}</span>{% endif %}</dd>
{# ── Image / placement ───────────────────────────────────────────────────── #}
<div class="card" style="margin-bottom:1.5rem;">
<div style="display:grid;grid-template-columns:max-content 1fr;gap:0.4rem 1.25rem;font-size:0.86rem;">
<span style="color:var(--text-muted);">Image</span>
<span style="font-family:ui-monospace,monospace;word-break:break-all;">{{ container.image or "—" }}</span>
{% if container.compose_project %}
<span style="color:var(--text-muted);">Compose project</span><span>{{ container.compose_project }}</span>
{% endif %}
{% if container.service_name %}
<span style="color:var(--text-muted);">Swarm service</span><span>{{ container.service_name }}</span>
{% endif %}
{% if container.node_id %}
<span style="color:var(--text-muted);">Node</span><span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>
{% endif %}
<dt>Memory limit</dt><dd>{% if usage.mem_limit %}{{ usage.mem_limit|bytes }}{% else %}<span class="tone-dim">none set</span>{% endif %}</dd>
{% if ports %}
<dt>Ports</dt><dd class="mono">{% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %}</dd>
<span style="color:var(--text-muted);">Ports</span>
<span style="font-family:ui-monospace,monospace;">{% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %}</span>
{% endif %}
{% if c.node_id %}
<dt>Swarm node</dt><dd title="node id {{ c.node_id }}">{{ node_name or c.node_id[:12] }}</dd>
{% endif %}
<dt>Network</dt><dd>{{ net_rx }} in · {{ net_tx }} out <span class="tone-dim">since it started</span></dd>
<dt>Disk</dt><dd>{{ blk_read }} read · {{ blk_write }} written <span class="tone-dim">since it started</span></dd>
</dl>
</div>
</div>
{# ── Resource history: the one home for what it uses ────────────────────── #}
<section class="card" aria-labelledby="ctr-history">
{# ── Resource history (range-toggled) ────────────────────────────────────── #}
<div class="card" style="margin-bottom:1.5rem;">
<div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
<h2 id="ctr-history" class="section-title" style="margin:0;">Resource history</h2>
<h3 class="section-title" style="margin-bottom:0;">Resource history</h3>
{% include "_time_range.html" %}
</div>
<div id="container-history"
@@ -100,30 +80,31 @@
hx-trigger="load, rangeChange from:body"
hx-include="#time-range"
hx-swap="innerHTML">
<p class="field-hint">Loading…</p>
<div style="color:var(--text-muted);font-size:0.85rem;">Loading…</div>
</div>
</section>
</div>
{# ── Lifecycle, newest first ─────────────────────────────────────────────── #}
<section class="card-flush" aria-labelledby="ctr-lifecycle">
<h2 id="ctr-lifecycle" class="table-caption">Lifecycle <span class="tone-muted">newest first</span></h2>
{# ── Lifecycle timeline ──────────────────────────────────────────────────── #}
<div class="card">
<h3 class="section-title" style="margin-bottom:0.75rem;">Lifecycle</h3>
{% if events %}
<table class="table">
<thead><tr><th>Event</th><th>Detail</th><th class="num">When</th></tr></thead>
<tbody>
{% for e in events %}
<tr>
<td><span class="tone-{{ e.tone }}" aria-hidden="true">{{ e.glyph }}</span> {{ e.event|replace("_", " ") }}</td>
<td class="tone-muted" style="overflow-wrap:anywhere;">{{ e.detail or "" }}</td>
<td class="num">{{ e.at|ago }}</td>
</tr>
{% endfor %}
</tbody>
</table>
<div style="display:grid;gap:0.5rem;">
{% for e in events %}
<div style="display:flex;align-items:baseline;gap:0.6rem;font-size:0.85rem;">
<span style="color:{{ e.colour }};width:1rem;text-align:center;flex-shrink:0;">{{ e.glyph }}</span>
<span style="font-weight:500;width:6.5rem;flex-shrink:0;">{{ e.event }}</span>
<span style="color:var(--text-muted);flex:1;min-width:0;">{{ e.detail or "" }}</span>
<span style="color:var(--text-dim);font-size:0.78rem;white-space:nowrap;" title="{{ e.at }}">{{ e.at.strftime("%Y-%m-%d %H:%M") }}</span>
</div>
{% endfor %}
</div>
{% else %}
<p class="tone-muted" style="margin:0;padding:1rem;">No lifecycle events recorded yet. Starts, stops and health changes appear here as the agent reports them.</p>
<div style="color:var(--text-muted);font-size:0.85rem;">
No lifecycle events recorded yet. Start/stop/health changes appear here as the
agent reports them over time.
</div>
{% endif %}
</section>
</div>
{% endif %}
{% endblock %}
plugins/docker/templates/docker/container_logs.html
-51
View File
@@ -1,51 +0,0 @@
{# docker/container_logs.html — per-container log viewer (m79) #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs %}
{% block title %}Logs — {{ name }} — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([
("Docker", "/plugins/docker/"),
(name, "/plugins/docker/container/" ~ host_id ~ "/" ~ name),
("Logs", "")]) }}{% endblock %}
{% block content %}
<div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
<h1 class="page-title" style="margin-bottom:0;">{{ name }}</h1>
<span style="font-size:0.9rem;color:var(--text-muted);">logs</span>
</div>
<div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1rem;">
Recent lines (newest first) collected by the host agent{% if host %} on
<a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %} —
updated every few seconds.
<a href="/plugins/docker/container/{{ host_id }}/{{ name }}">← back to container</a>
</div>
{# ── Controls: stream filter + text search (drive the fragment via HTMX) ───── #}
<form id="log-controls" onsubmit="return false;"
style="display:flex;gap:0.6rem;align-items:center;flex-wrap:wrap;margin-bottom:0.6rem;">
<label style="font-size:0.8rem;color:var(--text-muted);display:flex;align-items:center;gap:0.35rem;">
Stream
<select name="stream" style="padding:0.25rem 0.4rem;">
<option value="all">all</option>
<option value="stdout">stdout</option>
<option value="stderr">stderr</option>
</select>
</label>
<input type="search" name="q" placeholder="Filter lines…" autocomplete="off"
aria-label="Filter log lines"
style="flex:1;min-width:180px;padding:0.3rem 0.5rem;">
</form>
<div class="card-flush">
<div id="log-lines"
hx-get="/plugins/docker/container/{{ host_id }}/{{ name }}/logs/lines"
hx-trigger="load, every 5s, change from:#log-controls, keyup changed delay:400ms from:#log-controls"
hx-include="#log-controls"
hx-swap="innerHTML"
role="log" aria-live="polite" tabindex="0"
style="max-height:70vh;overflow:auto;padding:0.5rem 0.75rem;
font-family:ui-monospace,monospace;font-size:0.8rem;line-height:1.5;">
<div style="color:var(--text-muted);">Loading…</div>
</div>
</div>
{% endblock %}
plugins/docker/templates/docker/disk.html
+63 -104
View File
@@ -1,124 +1,83 @@
{# docker/disk.html — what Docker keeps on each host's disk (#5304, #5305):
the hosts side by side with how much each grew, then each host's parts,
its cleanup actions, and its images folded under their repository. #}
{# docker/disk.html — image/disk usage: reclaimable space, per-image sizes #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header, stat_tile, status_badge %}
{% from "_macros.html" import crumbs %}
{% block title %}Disk — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Disk", "")]) }}{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Image & disk usage", "")]) }}{% endblock %}
{% block content %}
{% call page_header("Docker disk", "What Docker keeps on each host's disk, from docker system df, and how it has grown.") %}
{% include "_time_range.html" %}
{% endcall %}
<h1 class="page-title" style="margin-bottom:0.4rem;">Image &amp; disk usage</h1>
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
Reclaimable = space held by images no container references. Cleanup actions
(prune) arrive in a later release — these are read-only figures for now.
</p>
{% if host_groups %}
<section class="card-flush" aria-labelledby="disk-hosts">
<h2 id="disk-hosts" class="table-caption">By host <span class="tone-muted">most space first</span></h2>
<div hx-get="/plugins/docker/disk/hosts" hx-trigger="load, rangeChange from:body"
hx-include="#time-range" hx-swap="innerHTML">
<p class="field-hint" style="padding:1rem;margin:0;">Loading…</p>
</div>
</section>
{% for g in host_groups %}
{% set s = g.summary %}
<section id="host-{{ g.host_id }}" class="disk-host" aria-labelledby="disk-host-{{ g.host_id }}">
<h2 id="disk-host-{{ g.host_id }}" class="disk-host-head">
<a href="/hosts/{{ g.host_id }}">{{ g.host_name }}</a>
<span class="tone-muted">{{ g.total|bytes }}</span>
{% if g.as_of %}<span class="field-hint">as of {{ g.as_of|ago }}</span>{% endif %}
</h2>
<div style="margin-bottom:2rem;">
<div style="font-weight:600;font-size:0.95rem;margin-bottom:0.75rem;">{{ g.host_name }}</div>
<div class="stat-strip">
{{ stat_tile("Reclaimable", s.images_reclaimable|bytes, state="warn" if s.images_reclaimable else "ok", hint="held by images no container uses") }}
{{ stat_tile("Images", s.images_size|bytes, hint=s.images_active ~ " of " ~ s.images_total ~ " in use") }}
{{ stat_tile("Container files", s.containers_size|bytes, hint="written by " ~ s.containers_count ~ " containers on top of their images") }}
{{ stat_tile("Volumes", s.volumes_size|bytes, hint=s.volumes_count ~ " volumes; never pruned here") }}
{{ stat_tile("Build cache", s.build_cache_size|bytes, hint="left by docker build; safe to clear") }}
{{ stat_tile("Stopped", g.stopped, state="dim" if not g.stopped else none, hint="containers not running") }}
{# ── Summary stats ────────────────────────────────────────────────────── #}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1rem;">
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Reclaimable</div>
<span class="stat-val" style="color:{% if g.summary.images_reclaimable != '0 B' %}var(--orange){% else %}var(--green){% endif %};">{{ g.summary.images_reclaimable }}</span>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Images</div>
<span class="stat-val">{{ g.summary.images_size }}</span>
<div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.images_active }}/{{ g.summary.images_total }} in use</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Stopped</div>
<span class="stat-val" style="{% if g.stopped %}color:var(--text-muted){% endif %};">{{ g.stopped }}</span>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Writable layers</div>
<span class="stat-val">{{ g.summary.containers_size }}</span>
<div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.containers_count }} containers</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Volumes</div>
<span class="stat-val">{{ g.summary.volumes_size }}</span>
<div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.volumes_count }} volumes</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Build cache</div>
<span class="stat-val">{{ g.summary.build_cache_size }}</span>
</div>
</div>
{# ── Cleanup actions (admin only; audited Ansible prune run) ───────────── #}
{% if session.user_role == 'admin' %}
<div class="disk-actions">
{% if ansible_available and g.has_target %}
<form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune"
data-confirm="Remove all STOPPED containers on {{ g.host_name }}? This cannot be undone.">{{ csrf_field() }}
<input type="hidden" name="target" value="containers">
<button type="submit" class="btn btn-sm">Prune stopped containers</button>
</form>
<form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune"
data-confirm="Remove ALL unused images on {{ g.host_name }} (docker image prune -a)? Any image not used by a container is deleted.">{{ csrf_field() }}
<input type="hidden" name="target" value="images">
<button type="submit" class="btn btn-sm">Prune unused images</button>
</form>
<form method="post" action="/plugins/docker/disk/{{ g.host_id }}/prune"
data-confirm="Run a full system prune on {{ g.host_name }}? Removes stopped containers, unused networks, untagged images nothing uses, and the build cache.">{{ csrf_field() }}
<input type="hidden" name="target" value="system">
<button type="submit" class="btn btn-sm btn-danger">System prune…</button>
</form>
<span class="field-hint">Each runs an audited Ansible playbook on {{ g.host_name }}; reclaimed space shows on the next agent sample.</span>
{% elif not ansible_available %}
<span class="field-hint">Cleanup needs the Ansible runner (currently unavailable).</span>
{% else %}
<span class="field-hint">Link an <a href="/hosts/{{ g.host_id }}">Ansible target</a> to this host to enable prune actions.</span>
{% endif %}
</div>
{% endif %}
{# ── Images by repository, versions folded under it (#5305) ─────────── #}
<section class="card-flush" aria-label="Images on {{ g.host_name }}">
<h3 class="table-caption">Images by repository
<span class="tone-muted">{% if g.images_capped %}the {{ g.images_listed }} largest of {{ s.images_total }} images{% else %}{{ g.images_listed }} images{% endif %}</span></h3>
{% if g.repos %}
<table class="table image-repos">
<thead><tr>
<th>Repository</th>
<th class="num">Versions</th>
<th class="num" title="Each version's size less the layers it shares with other images, added up: what removing them all would free, roughly">Own size</th>
<th class="num">Used by</th>
<th class="num" title="Own size of the versions no container uses">Reclaimable</th>
</tr></thead>
{% for r in g.repos %}
<tbody class="image-repo">
{# ── Per-image table ──────────────────────────────────────────────────── #}
<div class="card-flush">
<table class="table">
<thead>
<tr><th>Image</th><th>Size</th><th>Shared</th><th>Containers</th></tr>
</thead>
<tbody>
{% for im in g.images %}
<tr>
<td class="cell-name">
<details>
<summary title="{{ r.repo or 'untagged images' }}">{% if r.repo %}<span class="mono">{{ r.repo }}</span>{% else %}<span class="tone-muted">untagged</span>{% endif %}</summary>
<table class="table image-versions">
<thead><tr><th>Tag</th><th>ID</th><th class="num">Size</th><th class="num" title="Layers in common with other images: stored once">Shared</th><th class="num">Used by</th></tr></thead>
<tbody>
{% for v in r.versions %}
<tr>
<td class="mono">{{ v.tag or "—" }}</td>
<td class="mono tone-muted">{{ (v.image_id or "")|replace("sha256:", "")|truncate(12, true, "") }}</td>
<td class="num">{{ v.size|bytes }}</td>
<td class="num tone-muted">{{ v.shared|bytes }}</td>
<td class="num">{% if v.containers %}{{ v.containers }} container{{ "s" if v.containers != 1 }}{% else %}<span class="tone-warn">nothing</span>{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
</details>
<td style="font-size:0.84rem;font-family:ui-monospace,monospace;max-width:340px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
{{ im.repo_tag }}
{% if im.reclaimable %}<span title="not referenced by any container" style="font-size:0.68rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0 0.3rem;margin-left:0.4rem;">reclaimable</span>{% endif %}
</td>
<td class="num">{{ r.versions|length }}{% if r.unused %} <span class="tone-dim">({{ r.unused }} unused)</span>{% endif %}</td>
<td class="num">{{ r.own|bytes }}</td>
<td class="num">{% if r.containers %}{{ r.containers }} container{{ "s" if r.containers != 1 }}{% else %}<span class="tone-warn">nothing</span>{% endif %}</td>
<td class="num">{% if r.reclaimable %}<span class="tone-warn">{{ r.reclaimable|bytes }}</span>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.86rem;">{{ im.size }}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.86rem;color:var(--text-muted);">{{ im.shared }}</td>
<td style="font-size:0.86rem;color:{% if im.containers %}var(--text){% else %}var(--text-muted){% endif %};">{{ im.containers }}</td>
</tr>
{% else %}
<tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No images reported.</td></tr>
{% endfor %}
</tbody>
{% endfor %}
</table>
{% else %}
<p class="tone-muted" style="margin:0;padding:1rem;">No images reported.</p>
{% endif %}
</section>
</section>
</div>
</div>
{% endfor %}
{% else %}
<div class="card empty-state">
No disk usage reported yet. The host agent reports what Docker keeps on disk,
from <code>docker system df</code>, on hosts running Docker.
<div class="card" style="text-align:center;padding:3rem;">
<div style="color:var(--text-muted);font-size:0.9rem;">
No disk usage reported yet. The host agent reports image/disk usage from
<code>docker system df</code> on hosts running Docker.
</div>
</div>
{% endif %}
{% endblock %}
plugins/docker/templates/docker/host_panel.html
+14 -68
View File
@@ -1,76 +1,22 @@
{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub:
problems first with what is wrong and since when, then the busiest running
containers; the rest of the running and the stopped fold away so the card
stays short (#5276). Tasks Swarm has replaced are left out (counted in the
header). #}
{% from "_macros.html" import status_badge, status_dot %}
{% from "docker/_usage.html" import cpu, mem %}
{% macro container_table(rows) %}
<table class="table">
<colgroup><col><col style="width:7rem"><col style="width:9rem"></colgroup>
<thead>
<tr>
<th>Container</th>
<th title="Docker counts CPU per core: 1 core = one core fully busy">CPU</th>
<th title="Used, and the limit when one is set">Memory</th>
</tr>
</thead>
<tbody>
{% for r in rows %}
{% set c = r.c %}
<tr>
<td>
<div style="display:flex;align-items:center;gap:0.5rem;min-width:0;">
{{ status_dot('crit' if r.problems else ('up' if c.status == 'running' else 'inactive'), c.status|capitalize) }}
<div style="min-width:0;">
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }} · {{ c.image }}" style="font-weight:500;color:inherit;text-decoration:none;overflow-wrap:anywhere;">{{ r.label }}</a>
{% if r.problems %}
{% set worst = r.problems[0] %}
<div style="font-size:0.75rem;margin-top:0.1rem;">
{{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
<span style="color:var(--text-muted);">
{%- for p in r.problems %} {{ says[p] }}{{ " and" if not loop.last }}{% endfor %}
{%- if c.exit_code %} · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}{% endif %}
{%- if r.since %} · since {{ r.since|ago }}{% endif -%}
</span>
</div>
{% else %}
<div style="font-size:0.72rem;color:var(--text-muted);">
{{ c.status }}{% if r.uptime %} · up {{ r.uptime }}{% endif %}
{%- if c.status != 'running' and c.exit_code %} · exit {{ c.exit_code }}{% endif %}
</div>
{% endif %}
</div>
</div>
</td>
<td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.85rem;white-space:nowrap;">{% if c.status == 'running' %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endmacro %}
{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub #}
<div class="card">
<div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;flex-wrap:wrap;">
<div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;">
<h3 class="section-title" style="margin-bottom:0;">Docker</h3>
<span style="font-size:0.78rem;color:var(--text-muted);">
{% if problems %}<span class="tone-crit">{{ problems }} with problems</span> · {% endif %}
{{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %}
{% if replaced %}<span title="Old swarm tasks Swarm has replaced with new ones: history, not containers that need anything"> · {{ replaced }} replaced task{{ "s" if replaced != 1 }} hidden</span>{% endif %}
<a href="/plugins/docker/" style="margin-left:0.6rem;color:var(--text-muted);">All →</a>
</span>
</div>
{{ container_table(rows) }}
{% if folded %}
<details style="margin-top:0.5rem;">
<summary style="cursor:pointer;font-size:0.82rem;color:var(--text-muted);">
{%- set more_running = folded|selectattr("c.status", "equalto", "running")|list|length -%}
{%- set more_stopped = folded|length - more_running -%}
{%- if more_running %}+{{ more_running }} more running{% endif -%}
{%- if more_running and more_stopped %} · {% endif -%}
{%- if more_stopped %}{{ more_stopped }} stopped{% endif -%}
</summary>
{{ container_table(folded) }}
</details>
{% endif %}
<div style="display:grid;gap:2px;">
{% for c in containers %}
<div style="display:flex;align-items:center;gap:0.5rem;font-size:0.85rem;padding:0.2rem 0;">
<span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;" title="{{ c.image }}">{{ c.name }}</a>
<span style="font-size:0.74rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
{% if c.cpu_pct is not none %}{{ "%.1f" | format(c.cpu_pct) }}%{% endif %}
{% if c.mem_pct is not none %} · {{ "%.1f" | format(c.mem_pct) }}%{% endif %}
</span>
</div>
{% endfor %}
</div>
</div>
plugins/docker/templates/docker/index.html
+15 -72
View File
@@ -1,81 +1,24 @@
{% extends "base.html" %}
{% from "_macros.html" import page_header %}
{# The Docker page. The filter bar sits outside the polled fragment, so a poll
never resets what is being typed; the fragment reads the bar on every poll,
and a change re-fetches it and puts the filters in the URL (#5299). #}
{% block title %}Docker — Steward{% endblock %}
{% block content %}
{% call page_header("Docker", "What needs attention first, then every service and container.") %}
{% if has_swarm %}<a class="btn btn-sm btn-ghost" href="/plugins/docker/swarm">Swarm</a>{% endif %}
{% if has_disk %}<a class="btn btn-sm btn-ghost" href="/plugins/docker/disk">Disk</a>{% endif %}
{% endcall %}
<form id="docker-filters" class="filter-bar" method="get" action="/plugins/docker/" role="search">
<input type="search" id="docker-q" name="q" value="{{ f.q }}" class="input"
placeholder="Search name, image or stack" aria-label="Search services and containers" autocomplete="off">
{% if choices.stacks %}
<select name="stack" aria-label="Stack">
<option value="">All stacks</option>
{% for s in choices.stacks %}<option value="{{ s }}"{% if f.stack == s %} selected{% endif %}>{{ s }}</option>{% endfor %}
<option value="{{ no_stack }}"{% if f.stack == no_stack %} selected{% endif %}>No stack</option>
</select>
{% endif %}
{% if choices.hosts|length > 1 %}
<select name="host" aria-label="Host or node">
<option value="">All hosts</option>
{% for h in choices.hosts %}<option value="{{ h }}"{% if f.host == h %} selected{% endif %}>{{ h }}</option>{% endfor %}
</select>
{% endif %}
<select name="state" aria-label="Status">
<option value="">Any status</option>
{% for key, label in states.items() %}<option value="{{ key }}"{% if f.state == key %} selected{% endif %}>{{ label }}</option>{% endfor %}
</select>
{% if f.sort != 'status' %}<input type="hidden" name="sort" value="{{ f.sort }}"><input type="hidden" name="dir" value="{{ f.dir }}">{% endif %}
<noscript><button type="submit" class="btn btn-sm">Filter</button></noscript>
</form>
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1.5rem;gap:1rem;flex-wrap:wrap;">
<div style="display:flex;align-items:baseline;gap:1rem;">
<h1 class="page-title" style="margin-bottom:0;">Docker</h1>
{% if has_swarm %}
<a href="/plugins/docker/swarm" style="font-size:0.85rem;color:var(--accent);text-decoration:none;">Swarm →</a>
{% endif %}
{% if has_disk %}
<a href="/plugins/docker/disk" style="font-size:0.85rem;color:var(--accent);text-decoration:none;">Disk →</a>
{% endif %}
</div>
{% include "_time_range.html" %}
</div>
<div id="docker-rows"
hx-get="/plugins/docker/rows"
hx-include="#docker-filters"
hx-trigger="load, every {{ poll_interval }}s, change from:#docker-filters, input changed delay:250ms from:#docker-q, search from:#docker-q"
hx-trigger="load, rangeChange from:body"
hx-include="#time-range"
hx-swap="innerHTML">
<div class="card"><p class="field-hint">Loading…</p></div>
<div style="color:var(--text-muted);font-size:0.9rem;padding:2rem;">Loading...</div>
</div>
{% if disk %}
{# What Docker keeps on each host's disk, and its growth over a week (#5306). #}
<p class="docker-disk-line">
<a href="/plugins/docker/disk" class="section-title">Disk</a>
{% for d in disk %}
<span class="docker-disk-host"><a href="/plugins/docker/disk#host-{{ d.host_id }}">{{ d.host_name }}</a>
{{ d.total|bytes }}
{%- if d.growth and d.growth.change %} <span class="tone-{{ 'warn' if d.growth.pct is not none and d.growth.pct >= 10 else 'muted' }}"
title="Since {{ d.growth.since.strftime('%Y-%m-%d %H:%M UTC') }}">{{ "+" if d.growth.change > 0 else "−" }}{{ (d.growth.change|abs)|bytes }} in 7d</span>{% endif -%}
{%- if d.reclaimable %} <span class="tone-dim">· {{ d.reclaimable|bytes }} reclaimable</span>{% endif %}</span>
{% endfor %}
</p>
{% endif %}
<script>
(function () {
var rows = document.getElementById('docker-rows');
var form = document.getElementById('docker-filters');
// The filters live in the URL too, so a filtered view can be bookmarked and
// Back returns to it.
function keepInUrl() {
var q = new URLSearchParams(new FormData(form));
Array.from(q.keys()).forEach(function (k) { if (!q.get(k)) q.delete(k); });
var s = q.toString();
history.replaceState(null, '', location.pathname + (s ? '?' + s : '') + location.hash);
}
form.addEventListener('change', keepInUrl);
form.addEventListener('input', keepInUrl);
// A link to a section (the dashboard's "failed (24h)" → #failures) lands
// before the rows exist; scroll to it once, when they first arrive.
if (!location.hash || !rows) return;
rows.addEventListener('htmx:afterSwap', function once() {
rows.removeEventListener('htmx:afterSwap', once);
var target = document.getElementById(location.hash.slice(1));
if (target) target.scrollIntoView({ block: 'start' });
});
})();
</script>
{% endblock %}
plugins/docker/templates/docker/rows.html
+170 -224
View File
@@ -1,228 +1,174 @@
{# docker/rows.html — HTMX fragment for the Docker page (#5298–#5301):
a verdict line; what needs
attention (service.needs_attention, the ranking the MCP tools use); hosts
whose Docker is not answering; what failed in 24h, each with its logs; then
the swarm services and the standalone containers as two tables that share
the filter bar's search, filters and sort (overview.py). #}
{% from "_macros.html" import status_badge, status_dot, sort_th %}
{% from "docker/_usage.html" import cpu, mem, cores %}
{% macro th(key, label, cls=none, title=none) -%}
{{ sort_th(label, sort_url(key), active=f.sort == key, desc=f.dir == 'desc', cls=cls, title=title) }}
{%- endmacro %}
{% macro up(row) -%}
{%- if row.started_at %}<span title="Started {{ row.started_at.strftime('%Y-%m-%d %H:%M UTC') }}">{{ row.started_at|ago|striptags|replace(" ago", "") }}</span>
{%- else %}<span class="tone-dim">—</span>{% endif -%}
{%- endmacro %}
{# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #}
{# ── Verdict ──────────────────────────────────────────────────────────────── #}
{% set problems = attention|length + failing_services|length %}
<p class="page-verdict">
{% if problems %}<strong class="tone-crit">{{ problems }} need{{ "s" if problems == 1 }} attention</strong>
{% else %}<strong class="tone-ok">Nothing needs attention</strong>{% endif %}
{%- if services_total %} · <span class="tone-{{ 'ok' if services_healthy == services_total else 'warn' }}">{{ services_healthy }}/{{ services_total }}</span> services healthy{% endif %}
{%- if containers_total %} · {{ running }} container{{ "s" if running != 1 }} running{% if stopped %}, {{ stopped }} stopped{% endif %}{% if services_total %} outside services{% endif %}{% endif %}
{%- if docker_hosts %} · {{ docker_hosts }} Docker host{{ "s" if docker_hosts != 1 }}{% endif %}
{%- if stale %} · <span class="tone-warn" title="No recent report from their agent: what is shown may no longer be true">{{ stale }} stale</span>{% endif %}
</p>
{# ── Needs attention ──────────────────────────────────────────────────────── #}
{% if attention or failing_services %}
<section class="card problem-card" aria-labelledby="docker-problems">
<h2 id="docker-problems" class="section-title section-count">Needs attention <span class="tone-crit">{{ problems }}</span>
{%- if unhealthy %}<span class="tone-muted"> · {{ unhealthy }} unhealthy</span>{% endif %}
{%- if restarting %}<span class="tone-muted"> · {{ restarting }} restarting</span>{% endif %}</h2>
<ul class="problem-list">
{# A swarm service in trouble: Swarm replaces its failed tasks, so the
service, not any one container, is what is failing. #}
{% for s in failing_services %}
{% set e = s.last_error %}
<li>
{% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
{% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}
{% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
<a href="/plugins/docker/service/{{ s.name }}">{{ s.name }}</a>
<span class="problem-detail">
{{- s.running }} of {{ s.desired }} running
{%- if s.crash_looping %} · {{ s.failures_1h }} tasks failed in the last hour{% endif %}
{%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.exit_code %} (exit {{ e.exit_code }}){% endif %}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
{%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
</span>
<a class="field-hint" href="/plugins/docker/service/{{ s.name }}{% if e and e.task_id %}?task={{ e.task_id }}{% endif %}#logs" style="margin-left:0.4rem;">Logs →</a>
</li>
{% endfor %}
{% for r in attention %}
{% set c = r.container %}
{% set worst = r.problems[0] %}
<li>
{{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" title="{{ c.name }}">{{ r.label }}</a>
<span class="problem-detail">
{%- for p in r.problems %}{{ says[p] }}{{ " and " if not loop.last }}{% endfor %}
{%- if c.exit_code and worst in ('exited_nonzero', 'crash_looping', 'restarting') %} · exit {{ c.exit_code }}{% endif %}
{%- if r.since %} · since {{ r.since|ago }}{% endif %}
{%- if r.stale %} · {{ status_badge('stale') }} last report {{ c.scraped_at|ago if c.scraped_at else "never" }}{% endif -%}
</span>
<a class="field-hint" href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}/logs" style="margin-left:0.4rem;">Logs →</a>
</li>
{% endfor %}
</ul>
</section>
{% endif %}
{# ── Hosts whose Docker is not answering (agent ≥ 1.9.0) ──────────────────── #}
{% if docker_down %}
<section class="card problem-card" aria-labelledby="docker-unreachable">
<h2 id="docker-unreachable" class="section-title" style="margin-bottom:0.6rem;">Docker not answering</h2>
<ul class="problem-list">
{% for d in docker_down %}
<li>
{{ status_badge('crit', 'Error') if d.status == 'error' else status_badge('stale', 'No Docker') }}
<a href="/hosts/{{ d.host_id }}">{{ d.host_name }}</a>
<span class="problem-detail">{{ d.detail or "no reason given" }} · since {{ d.since|ago }} · its containers below are what Docker last reported</span>
</li>
{% endfor %}
</ul>
</section>
{% endif %}
{# ── What failed in the last 24h: what the dashboard's "failed (24h)" counts.
A plain container's death, or a swarm service's failed tasks; a container
someone stopped (exit 137/143) is not a failure. Each links to the logs of
what died: a service's to its failed task's lines (#5300). ───────────── #}
{% if failures %}
<section id="failures" class="card" aria-labelledby="docker-failures">
<h2 id="docker-failures" class="section-title section-count">Failed in the last 24h <span class="tone-warn">{{ failures|length }}</span></h2>
<ul class="problem-list">
{% for x in failures %}
<li>
{{ status_badge('warn', 'Service' if x.kind == 'service' else 'Container') }}
{% if x.kind == 'service' %}<a href="/plugins/docker/service/{{ x.name }}">{{ x.name }}</a>
{% else %}<a href="/plugins/docker/container/{{ x.host_id }}/{{ x.name }}">{{ x.name }}</a>{% endif %}
<span class="problem-detail">
{{- x.count }} {{ "failure" if x.count == 1 else "failures" }}
{%- if x.detail %} · last: {{ x.detail }}{% endif %}
{%- if x.host %} · {{ x.host }}{% endif %}
{%- if x.at %} · {{ x.at|ago }}{% endif -%}
</span>
{% if x.kind == 'service' %}
<a class="field-hint" href="/plugins/docker/service/{{ x.name }}{% if x.task_id %}?task={{ x.task_id }}{% endif %}#logs" style="margin-left:0.4rem;">Logs{% if x.task_id %} of that task{% endif %} →</a>
{% else %}
<a class="field-hint" href="/plugins/docker/container/{{ x.host_id }}/{{ x.name }}/logs" style="margin-left:0.4rem;">Logs →</a>
{% endif %}
</li>
{% endfor %}
</ul>
</section>
{% endif %}
{# ── Swarm services: one row each, problems first unless a header picked
another order; CPU and memory add up the running replicas (#5299). ───── #}
{% if services %}
<section class="card-flush" aria-labelledby="docker-services">
<h2 id="docker-services" class="table-caption">Swarm services <span class="tone-muted">{{ services|length }}{% if services|length != services_total %} of {{ services_total }}{% endif %}</span>
<a href="/plugins/docker/swarm" class="field-hint">By node →</a></h2>
<table class="table">
<thead>
<tr>
{{ th("name", "Service") }}
{{ th("status", "Status", title="Problems first: a crash loop, then down, then short of replicas, then failed tasks in 24h") }}
{{ th("stack", "Stack", title="The part of the service's name before the first _: the stack it was deployed in") }}
<th title="The nodes running its replicas">Nodes</th>
{{ th("cpu", "CPU", "num", "Across its running replicas; Docker counts CPU per core") }}
{{ th("mem", "Memory", "num", "Across its running replicas") }}
{{ th("up", "Up", "num", "Since its newest replica started") }}
<th class="num" title="Tasks that failed in the last 24h; Swarm replaced them">Failed 24h</th>
</tr>
</thead>
<tbody>
{% for r in services %}
<tr>
<td class="cell-name"><a href="/plugins/docker/service/{{ r.name }}" title="{{ r.name }}{% if r.image %} · {{ r.image }}{% endif %}">{{ r.name }}</a></td>
<td>
{{ status_dot('up' if r.state == 'healthy' else ('warn' if r.state == 'degraded' else 'down')) }}
<span class="tone-{{ 'ok' if r.state == 'healthy' else ('warn' if r.state == 'degraded' else 'crit') }}" style="font-variant-numeric:tabular-nums;">{{ r.running }}/{{ r.desired }}</span>
{%- if r.crash_looping %} {{ status_badge('crit', 'Crash-looping') }}{% elif r.unhealthy %} {{ status_badge('crit', 'Unhealthy') }}{% endif %}
{%- if r.mode == 'global' %} <span class="tone-dim">global</span>{% endif %}
</td>
<td>{% if r.stack %}<a class="chip" href="{{ page_url(stack=r.stack) }}" title="Show the {{ r.stack }} stack">{{ r.stack }}</a>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="tone-muted">
{{- r.nodes|join(", ") or ("" if r.ghost_nodes else "—") -}}
{%- if r.ghost_nodes %}{{ ", " if r.nodes }}<span title="Running on a node with no Steward agent: no figures for it">{{ r.ghost_nodes|join(", ") }} (no agent)</span>{% endif -%}
</td>
<td class="num">{% if r.cpu_cores is not none %}{{ cores(r.cpu_cores) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{% if r.mem_bytes is not none %}{{ r.mem_bytes|bytes }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{{ up(r) }}</td>
<td class="num">
{%- if r.failures_24h %}<a class="tone-warn" href="/plugins/docker/service/{{ r.name }}#tasks"
title="{% if r.last_error %}Last: {{ r.last_error.error or r.last_error.message or r.last_error.state }}{% if r.last_error.exit_code %} (exit {{ r.last_error.exit_code }}){% endif %}{% endif %}">{{ r.failures_24h }}</a>
{%- else %}<span class="tone-dim">—</span>{% endif -%}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</section>
{% endif %}
{# ── Containers outside any swarm service, every host in one table (#5301);
a compose project is its stack. ─────────────────────────────────────── #}
{% if containers %}
<section class="card-flush" aria-labelledby="docker-containers">
<h2 id="docker-containers" class="table-caption">{{ "Other containers" if services_total else "Containers" }} <span class="tone-muted">{{ containers|length }}{% if containers|length != containers_total %} of {{ containers_total }}{% endif %}</span></h2>
<table class="table">
<thead>
<tr>
{{ th("name", "Container") }}
{{ th("status", "Status", title="Problems first: unhealthy or restarting, then died with a failing exit, then stale; then running, then stopped") }}
<th>Host</th>
{{ th("stack", "Stack", title="Its compose project") }}
<th>Image</th>
{{ th("cpu", "CPU", "num", "Docker counts CPU per core: 1 core = one core fully busy") }}
{{ th("mem", "Memory", "num", "Used, and the limit when one is set") }}
{{ th("up", "Up", "num", "Since it started") }}
</tr>
</thead>
<tbody>
{% for r in containers %}
<tr>
<td class="cell-name">
<a href="/plugins/docker/container/{{ r.host_id }}/{{ r.container_name }}" title="{{ r.container_name }}">{{ r.name }}</a>
{%- if r.ports %}<div class="cell-sub mono">{% for p in r.ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{{ " " if not loop.last }}{% endfor %}</div>{% endif %}
</td>
<td>
{{ status_dot('stale' if r.stale else ('up' if r.running else ('pending' if r.status == 'paused' else 'down'))) }}
{%- if r.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}
{%- elif r.health == 'starting' %} {{ status_badge('pending', 'Starting') }}
{%- else %} <span class="tone-muted">{{ r.status }}</span>{% endif %}
{%- if not r.running and r.exit_code is not none and r.exit_code != 0 %}
<span class="tone-{{ 'crit' if r.exit_failed else 'dim' }}" title="{{ 'last exit code' if r.exit_failed else 'stopped by a signal; not a failure' }}">exit {{ r.exit_code }}{% if r.oom_killed %} (OOM){% endif %}</span>
{%- endif %}
{%- if r.restart_count %} <span class="tone-warn" title="Restarted {{ r.restart_count }} times">⟳{{ r.restart_count }}</span>{% endif %}
{%- if r.stale %} {{ status_badge('stale', title="Last report " ~ (r.scraped_at.strftime('%Y-%m-%d %H:%M UTC') if r.scraped_at else 'never') ~ "; what is shown may no longer be true") }}{% endif %}
</td>
<td class="tone-muted">{{ r.host }}</td>
<td>{% if r.stack %}<a class="chip" href="{{ page_url(stack=r.stack) }}" title="Show the {{ r.stack }} stack">{{ r.stack }}</a>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="cell-image" title="{{ r.image }}">{{ r.image }}</td>
<td class="num">{% if r.running %}{{ cpu(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{% if r.running %}{{ mem(r.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{{ up(r) }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</section>
{% endif %}
{% if not services and not containers %}
<div class="card">
{% if services_total or containers_total %}
<p class="tone-muted" style="margin:0;">Nothing matches these filters. <a href="/plugins/docker/">Show everything</a></p>
{% else %}
<p class="tone-muted" style="margin:0;">
No containers reported yet. Containers are collected by the Steward host agent —
deploy the agent to a host running Docker (Hosts → the host → agent panel) and
its containers will appear here.
</p>
{# ── Summary strip ─────────────────────────────────────────────────────────── #}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(140px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Running</div>
<span class="stat-val" style="color:var(--green);">{{ running }}</span>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Stopped</div>
<span class="stat-val" style="{% if stopped %}color:var(--text-muted){% endif %};">{{ stopped }}</span>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Total</div>
<span class="stat-val">{{ total }}</span>
</div>
{% if swarm_services %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Services</div>
<span class="stat-val">{{ swarm_services | length }}</span>
</div>
{% endif %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Hosts</div>
<span class="stat-val">{{ host_groups | length }}</span>
</div>
</div>
{# ── Swarm services: collapsed per-service, each replica labelled with its host.
Ghost replicas (tasks on nodes with no Steward agent) come from the managers'
placement data so the picture is cluster-complete. ──────────────────────── #}
{% if swarm_services %}
<div class="card-flush" style="margin-bottom:1.5rem;">
<div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
<span style="font-weight:600;font-size:0.95rem;">Swarm services</span>
<a href="/plugins/docker/swarm" style="font-size:0.78rem;color:var(--text-muted);">Topology →</a>
</div>
<div style="padding:0.5rem 0.75rem;display:grid;gap:0.85rem;">
{% for s in swarm_services %}
<div>
<div style="display:flex;align-items:center;gap:0.55rem;flex-wrap:wrap;margin-bottom:0.35rem;">
<span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
<span style="font-weight:600;font-size:0.9rem;">{{ s.name }}</span>
<span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
<span style="font-size:0.78rem;color:{% if s.state == 'healthy' %}var(--green){% elif s.state == 'degraded' %}var(--orange){% else %}var(--red){% endif %};font-variant-numeric:tabular-nums;">
{{ s.running }}/{{ s.desired }} running
</span>
</div>
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(210px,1fr));gap:0.4rem;padding-left:1.1rem;">
{% for r in s.replicas %}
{% if r.ghost %}
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:5px;padding:0.3rem 0.55rem;" title="Running on a node with no Steward agent — detail unavailable">
<span class="dot dot-warn" style="opacity:0.6;"></span>
<span style="font-weight:500;">{{ r.host }}</span>
<span style="color:var(--text-dim);margin-left:auto;">{{ r.count }} · no agent</span>
</div>
{% else %}
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px solid var(--border);border-radius:5px;padding:0.3rem 0.55rem;">
<span class="dot {% if r.status == 'running' %}dot-up{% elif r.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" style="font-weight:500;color:inherit;text-decoration:none;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" title="{{ r.name }}">{{ r.host }}</a>
{% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;">{{ "%.0f" | format(r.cpu_pct) }}%</span>{% endif %}
</div>
{% endif %}
{% endfor %}
</div>
</div>
{% endfor %}
</div>
</div>
{% endif %}
{# ── Container tables, one per host ───────────────────────────────────────── #}
{% if host_groups %}
{% for g in host_groups %}
<div class="card-flush" style="margin-bottom:1.5rem;">
<div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
{% if g.host %}
<a href="/hosts/{{ g.host.id }}" style="font-weight:600;font-size:0.95rem;color:inherit;text-decoration:none;">{{ g.host.name }}</a>
{% else %}
<span style="font-weight:600;font-size:0.95rem;">{{ g.host_name }}</span>
{% endif %}
<span style="font-size:0.78rem;color:var(--text-muted);">
{{ g.running }} running{% if g.stopped %} · {{ g.stopped }} stopped{% endif %}
</span>
</div>
<table class="table">
<thead>
<tr>
<th>Container</th>
<th>Image</th>
<th>Ports</th>
<th>CPU %</th>
<th style="min-width:100px;">CPU history</th>
<th>Mem %</th>
<th style="min-width:100px;">Mem history</th>
</tr>
</thead>
<tbody>
{% for sub in g.subgroups %}
{% if g.grouped and sub.label %}
<tr><td colspan="7" style="padding:0.4rem 0.75rem 0.2rem;font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;font-weight:600;">{{ sub.label }}</td></tr>
{% endif %}
{% for item in sub.containers %}
{% set c = item.container %}
<tr>
<td>
<div style="display:flex;align-items:center;gap:0.5rem;">
<span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<div>
<div style="font-weight:500;font-size:0.9rem;">
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="color:inherit;text-decoration:none;border-bottom:1px dotted var(--border-mid);">{{ c.name }}</a>
{% if c.health == 'healthy' %}<span title="healthy" style="color:var(--green);font-size:0.7rem;">●</span>
{% elif c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);font-size:0.7rem;">●</span>
{% elif c.health == 'starting' %}<span title="health: starting" style="color:var(--orange);font-size:0.7rem;">◐</span>{% endif %}
</div>
<div style="font-size:0.73rem;color:var(--text-muted);">
{{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
{% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
· <span style="color:var(--red);">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
{% endif %}
{% if c.restart_count %} · <span title="restart count" style="color:var(--orange);">⟳{{ c.restart_count }}</span>{% endif %}
</div>
{% if c.service_name or c.compose_project %}
<div style="font-size:0.68rem;color:var(--text-dim);margin-top:0.1rem;">
{{ c.service_name or c.compose_project }}
</div>
{% endif %}
</div>
</div>
</td>
<td style="font-size:0.82rem;color:var(--text-muted);max-width:200px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
{{ c.image }}
</td>
<td style="font-size:0.78rem;font-family:ui-monospace,monospace;">
{% for p in item.ports %}
<div style="color:var(--text-muted);">{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}</div>
{% endfor %}
</td>
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
{% if c.cpu_pct is not none %}
<span style="color:{% if c.cpu_pct > 80 %}var(--red){% elif c.cpu_pct > 50 %}var(--orange){% else %}var(--text){% endif %};">
{{ "%.1f" | format(c.cpu_pct) }}%
</span>
{% else %}
<span style="color:var(--text-dim);">—</span>
{% endif %}
</td>
<td>{{ item.sparkline_cpu | safe }}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
{% if c.mem_pct is not none %}
<span style="color:{% if c.mem_pct > 90 %}var(--red){% elif c.mem_pct > 70 %}var(--orange){% else %}var(--text){% endif %};">
{{ "%.1f" | format(c.mem_pct) }}%
</span>
{% else %}
<span style="color:var(--text-dim);">—</span>
{% endif %}
</td>
<td>{{ item.sparkline_mem | safe }}</td>
</tr>
{% endfor %}
{% endfor %}
</tbody>
</table>
</div>
{% endfor %}
{% elif not swarm_services %}
<div class="card" style="text-align:center;padding:3rem;">
<div style="color:var(--text-muted);font-size:0.9rem;">
No containers reported yet. Containers are collected by the Steward host agent —
deploy the agent to a host running Docker (Hosts → the host → agent panel) and
its containers will appear here under that host.
</div>
</div>
{% endif %}
plugins/docker/templates/docker/service.html
-160
View File
@@ -1,160 +0,0 @@
{# docker/service.html — one swarm service (#5302, #5310): a header that says
whether it is well and what it runs; its last failure only while it is in
trouble; its CPU and memory over time; one table of its tasks, the running
ones first with what they use; and the logs of every task it has run.
A failure's "Logs →" lands on #logs with ?task= picked. #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, status_badge %}
{% from "docker/_usage.html" import cpu, mem %}
{% block title %}{{ name }} — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
{% block content %}
{% if service is none %}
<div class="card empty-state">
<strong>{{ "No longer running" if task_choices else "Service not found" }}</strong>
{% if task_choices %}
<span>No swarm service named <code>{{ name }}</code> is reported now; it may have been removed.
The logs Steward still keeps from its tasks are below.</span>
{% else %}
<span>No swarm service named <code>{{ name }}</code> is reported, and Steward holds no logs or
events for one. It may have been removed. <a href="/plugins/docker/">Back to Docker</a></span>
{% endif %}
</div>
{% else %}
{# ── Header: the verdict, then what it is (#5310). Each fact has one home:
what it uses is in Resource history, its tasks and failures in Tasks. #}
{% set troubled = row.crash_looping or row.state != 'healthy' %}
{% set tone = 'ok' if row.state == 'healthy' else ('warn' if row.state == 'degraded' else 'crit') %}
{% set image = (svc.image if svc and svc.image else row.image) %}
{% set e = row.last_error %}
<div class="page-header">
<div style="min-width:0;">
<h1 class="page-title" style="overflow-wrap:anywhere;">{{ name }}</h1>
<p class="page-subtitle">
{% if row.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}
{% elif row.state == 'down' %}{{ status_badge('crit', 'Down') }}
{% elif row.state == 'degraded' %}{{ status_badge('warn', 'Degraded') }}
{% else %}{{ status_badge('ok', 'Healthy') }}{% endif %}
<span class="tone-{{ tone }}">{{ row.running }} of {{ row.desired }} running</span>
{%- if troubled and row.trouble_since %} · <span class="tone-{{ tone }}">in trouble since {{ row.trouble_since|ago }}</span>{% endif %}
· {{ row.mode }}
{%- if stack %} · stack <a class="chip" href="/plugins/docker/?stack={{ stack }}">{{ stack }}</a>{% endif %}
{%- if image %} · <span class="mono" style="overflow-wrap:anywhere;" title="{{ image }}">{{ image.split("@")[0] }}</span>{% endif %}
</p>
{% if (not troubled and e) or svc %}
<p class="page-subtitle tone-dim" style="margin-top:0.2rem;">
{%- if not troubled and e %}Last failed {% if e.age_s is not none %}{{ e.age_s|duration }} ago{% endif %}
{%- if e.exit_code %} (exit {{ e.exit_code }}){% endif %}
{%- if e.task_id %} · <a href="?task={{ e.task_id }}#logs">its logs</a>{% endif %} · {% endif -%}
{%- if svc %}reported {{ svc.updated_at|ago }} by the manager's agent{% endif -%}
</p>
{% endif %}
</div>
<div class="page-actions"><a class="btn btn-sm btn-ghost" href="#logs">Logs</a></div>
</div>
{# ── What is wrong, only while it is (#5310) ──────────────────────────────── #}
{% if troubled and e %}
<section class="card problem-card" aria-labelledby="svc-last-error">
<h2 id="svc-last-error" class="section-title" style="margin-bottom:0.4rem;">Last failed task</h2>
<p style="margin:0;">
{{ e.error or e.message or e.state }}{% if e.exit_code %} <span class="tone-crit">(exit {{ e.exit_code }})</span>{% endif %}
{%- if e.node %} · on {{ e.node }}{% endif %}{% if e.age_s is not none %} · {{ e.age_s|duration }} ago{% endif %}
{%- if e.task_id %} · <a href="?task={{ e.task_id }}#logs">its logs →</a>{% endif %}
</p>
</section>
{% endif %}
{# ── Resource history, summed across replicas ────────────────────────────── #}
<section class="card" aria-labelledby="svc-history">
<div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
<h2 id="svc-history" class="section-title" style="margin:0;">Resource history <span class="tone-dim">across running replicas</span></h2>
{% include "_time_range.html" %}
</div>
<div hx-get="/plugins/docker/service/{{ name }}/history"
hx-trigger="load, rangeChange from:body"
hx-include="#time-range"
hx-swap="innerHTML">
<p class="field-hint">Loading…</p>
</div>
</section>
{# ── Tasks: the running ones with what they use, then Swarm's record ─────── #}
<section id="tasks" class="card-flush" aria-labelledby="svc-tasks">
<h2 id="svc-tasks" class="table-caption">Tasks
<span class="tone-muted">running first, then Swarm's most recent</span>
{%- if row.failures_24h %} <span class="tone-warn">· {{ row.failures_24h }} failed in 24h</span>{% endif %}</h2>
{% if tasks %}
<table class="table">
<thead><tr>
<th title="The replica slot this task filled">Slot</th><th>Task</th><th>Node</th><th>State</th>
<th class="num" title="Docker counts CPU per core">CPU</th><th class="num">Memory</th>
<th class="num">Exit</th><th>Why</th><th class="num" title="When it started, for a running task; else when it reached its state">When</th>
</tr></thead>
<tbody>
{% for t in tasks %}
<tr>
<td>{{ t.slot or "—" }}</td>
<td class="mono" title="{{ t.container or t.task_id }}">
{%- if t.task_id %}<a href="?task={{ t.task_id }}#logs" title="Its logs">{{ t.task_id[:10] }}</a>
{%- if t.container and t.host_id %} <a class="tone-dim" href="/plugins/docker/container/{{ t.host_id }}/{{ t.container }}" title="The container's own page">↗</a>{% endif %}
{%- elif t.container and t.host_id %}<a href="/plugins/docker/container/{{ t.host_id }}/{{ t.container }}">{{ t.container|truncate(24, true, "…") }}</a>
{%- else %}<span class="tone-dim">—</span>{% endif -%}
</td>
<td>{% if t.host_id %}<a href="/hosts/{{ t.host_id }}">{{ t.node }}</a>{% else %}{{ t.node }}{% endif %}</td>
<td>
{%- if t.failed %}{{ status_badge('crit', t.state|capitalize) }}
{%- elif t.current %}{{ status_badge('ok', 'Running') }}
{%- else %}<span class="tone-muted">{{ t.state }}</span>{% endif %}
{%- if t.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}{% elif t.health == 'starting' %} {{ status_badge('pending', 'Starting') }}{% endif %}
{%- if t.desired and t.desired != t.state and not t.current %} <span class="tone-dim" title="What Swarm wants of it">→ {{ t.desired }}</span>{% endif -%}
</td>
<td class="num">{% if t.usage %}{{ cpu(t.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{% if t.usage %}{{ mem(t.usage) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{% if t.exit_code %}<span class="tone-{{ 'crit' if t.failed else 'dim' }}">{{ t.exit_code }}</span>{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td style="overflow-wrap:anywhere;">{{ t.why or "" }}</td>
<td class="num">{% if t.at %}{{ t.at|ago }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p class="tone-muted" style="margin:0;padding:1rem;">No task is running, and Swarm's record of its tasks has not been reported yet (the manager's agent reports it from version 1.8.0).</p>
{% endif %}
</section>
{% endif %}
{# ── Logs across every task, newest first ─────────────────────────────────
Shown for a removed service too, while its lines are still kept. #}
{% if service is not none or task_choices %}
<section id="logs" class="card-flush" aria-labelledby="svc-logs">
<h2 id="svc-logs" class="table-caption">Logs <span class="tone-muted">every task, newest first</span></h2>
<form id="svc-log-controls" class="filter-bar" onsubmit="return false;" style="padding:0.75rem 1rem 0;">
<input type="search" name="q" class="input" placeholder="Filter lines…" autocomplete="off" aria-label="Filter log lines">
{% if task_choices|length > 1 %}
<select name="task" aria-label="Task">
<option value="">All tasks</option>
{% for tid, label in task_choices %}<option value="{{ tid }}"{% if tid == focus_task %} selected{% endif %}>{{ label }}</option>{% endfor %}
</select>
{% elif focus_task %}<input type="hidden" name="task" value="{{ focus_task }}">{% endif %}
<select name="stream" aria-label="Stream">
<option value="all">stdout and stderr</option>
<option value="stdout">stdout</option>
<option value="stderr">stderr</option>
</select>
</form>
<div id="svc-log-lines"
hx-get="/plugins/docker/service/{{ name }}/logs/lines"
hx-trigger="load, every 5s, change from:#svc-log-controls, keyup changed delay:400ms from:#svc-log-controls"
hx-include="#svc-log-controls"
hx-swap="innerHTML"
role="log" aria-live="polite" tabindex="0"
style="max-height:60vh;overflow:auto;padding:0.5rem 1rem 0.75rem;font-family:var(--fs-font-mono);font-size:0.8rem;line-height:1.5;">
<p class="field-hint">Loading…</p>
</div>
</section>
{% endif %}
{% endblock %}
plugins/docker/templates/docker/swarm.html
+73 -89
View File
@@ -1,103 +1,87 @@
{# docker/swarm.html — the Swarm laid out by node (#5303). A verdict, the
services short of replicas, then one card per node: a node that is down or
not taking work first, then managers. Each card says what its containers use
against what the machine has, and lists the services placed on it. #}
{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, status_badge, status_dot %}
{% from "docker/_usage.html" import cores as fmt_cores %}
{% from "_macros.html" import crumbs %}
{% block title %}Swarm — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Swarm", "")]) }}{% endblock %}
{% block content %}
<h1 class="page-title">Swarm</h1>
<h1 class="page-title" style="margin-bottom:1.5rem;">Swarm</h1>
{% if swarms %}
{% for g in swarms %}
{% set down_nodes = g.nodes|rejectattr("ok")|list %}
<section class="swarm-group" aria-label="Swarm reported by {{ g.managers|join(', ') }}">
<p class="page-verdict">
{% if down_nodes or g.troubled %}<strong class="tone-crit">{{ down_nodes|length + g.troubled|length }} need{{ "s" if (down_nodes|length + g.troubled|length) == 1 }} attention</strong>
{% else %}<strong class="tone-ok">Every node is up and taking work</strong>{% endif %}
· {{ g.nodes|length }} node{{ "s" if g.nodes|length != 1 }}
· <span class="tone-{{ 'ok' if g.services_healthy == g.services_total else 'warn' }}">{{ g.services_healthy }}/{{ g.services_total }}</span> services healthy
<span class="tone-dim">· reported by {{ g.managers|join(", ") }}</span>
</p>
{% if down_nodes or g.troubled %}
<section class="card problem-card" aria-label="Needs attention">
<ul class="problem-list">
{% for n in down_nodes %}
<li>
{{ status_badge('crit', 'Node ' ~ n.status) if n.status != 'ready' else status_badge('warn', n.availability|capitalize) }}
<span>{{ n.hostname }}</span>
<span class="problem-detail">{{ n.role }} · {{ n.task_count }} task{{ "s" if n.task_count != 1 }} placed on it</span>
</li>
{% endfor %}
{% for s in g.troubled %}
<li>
{% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}{% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}{% else %}{{ status_badge('warn', 'Degraded') }}{% endif %}
<a href="/plugins/docker/service/{{ s.name }}">{{ s.name }}</a>
{% set e = s.last_error %}
<span class="problem-detail">
{{- s.running }} of {{ s.desired }} running
{%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.node %} on {{ e.node }}{% endif %}{% endif %}
{%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%}
</span>
</li>
{% endfor %}
</ul>
</section>
{% endif %}
<div class="node-grid">
{% for n in g.nodes %}
<section class="card node-card{% if not n.ok %} problem-card{% endif %}" aria-labelledby="node-{{ n.node_id }}">
<h2 id="node-{{ n.node_id }}" class="node-card-head">
{{ status_dot('up' if n.status == 'ready' else 'down') }}
{% if n.host_id %}<a href="/hosts/{{ n.host_id }}">{{ n.hostname }}</a>{% else %}{{ n.hostname }}{% endif %}
<span class="chip">{{ "leader" if n.leader else n.role }}</span>
{% if n.availability != 'active' %}{{ status_badge('warn', n.availability|capitalize, title="Swarm places no new tasks here") }}{% endif %}
</h2>
<p class="node-card-figures">
{% if n.cores_used is not none %}
Containers use <strong>{{ fmt_cores(n.cores_used) }}</strong>{% if n.cores %} of {{ n.cores }}{% endif %}
· <strong>{{ n.mem_used|bytes }}</strong>{% if n.mem %} of {{ n.mem|bytes }}{% endif %}
{% elif n.agent %}
<span class="tone-muted">No swarm task reports figures here yet.</span>
{% else %}
<span class="tone-muted" title="Install the Steward agent on this node to see what its tasks use">No Steward agent: what runs here is Swarm's placement, without figures.</span>
{% endif %}
</p>
{% if n.tasks %}
<table class="table node-tasks">
<thead><tr><th>Service</th><th class="num">Tasks</th><th class="num">CPU</th><th class="num">Memory</th></tr></thead>
<tbody>
{% for t in n.tasks %}
<tr>
<td class="cell-name">
{%- if t.state != 'healthy' %}{{ status_dot('down' if t.state == 'down' else 'warn') }} {% endif -%}
<a href="/plugins/docker/service/{{ t.name }}" title="{{ t.name }}">{{ t.name }}</a>
</td>
<td class="num">{{ t.tasks }}</td>
<td class="num">{% if t.cores is not none %}{{ fmt_cores(t.cores) }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
<td class="num">{% if t.mem_bytes is not none %}{{ t.mem_bytes|bytes }}{% else %}<span class="tone-dim">—</span>{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p class="tone-muted" style="margin:0;">No tasks placed here.</p>
{% endif %}
</section>
{% endfor %}
<div style="margin-bottom:2rem;">
<div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.75rem;flex-wrap:wrap;">
<span style="font-weight:600;font-size:0.95rem;">Swarm</span>
<span style="font-size:0.78rem;color:var(--text-muted);">
reported by {{ g.managers | length }} manager{{ 's' if g.managers | length != 1 }} · {{ g.managers | join(", ") }}
</span>
</div>
</section>
{# ── Services ─────────────────────────────────────────────────────────── #}
<div class="card-flush" style="margin-bottom:1rem;">
<table class="table">
<thead>
<tr>
<th>Service</th><th>Mode</th><th>Replicas</th><th>Image</th><th>Placement</th>
</tr>
</thead>
<tbody>
{% for s in g.services %}
<tr>
<td style="font-weight:500;font-size:0.9rem;">{{ s.name }}</td>
<td style="font-size:0.82rem;color:var(--text-muted);">{{ s.mode }}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
<span style="color:{% if s.healthy %}var(--green){% elif s.down %}var(--red){% elif s.degraded %}var(--orange){% else %}var(--text-muted){% endif %};">
{{ s.running }}/{{ s.desired }}
</span>
</td>
<td style="font-size:0.8rem;color:var(--text-muted);max-width:220px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.image or "—" }}</td>
<td style="font-size:0.78rem;color:var(--text-muted);">
{% for p in s.placement %}{{ p.node }} ({{ p.running }}){% if not loop.last %}, {% endif %}{% else %}—{% endfor %}
</td>
</tr>
{% else %}
<tr><td colspan="5" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No services in this swarm.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
{# ── Nodes ────────────────────────────────────────────────────────────── #}
<div class="card-flush">
<table class="table">
<thead>
<tr><th>Node</th><th>Role</th><th>Availability</th><th>Status</th></tr>
</thead>
<tbody>
{% for n in g.nodes %}
<tr>
<td style="font-weight:500;font-size:0.9rem;">
{{ n.hostname or n.node_id }}
{% if n.leader %}<span title="cluster leader" style="font-size:0.68rem;color:var(--accent);border:1px solid var(--accent);border-radius:3px;padding:0 0.3rem;margin-left:0.3rem;">leader</span>{% endif %}
</td>
<td style="font-size:0.82rem;color:var(--text-muted);">{{ n.role }}</td>
<td style="font-size:0.82rem;color:{% if n.availability == 'active' %}var(--text){% else %}var(--orange){% endif %};">{{ n.availability }}</td>
<td style="font-size:0.82rem;">
<span class="dot {% if n.status == 'ready' %}dot-up{% else %}dot-down{% endif %}"></span>
<span style="color:{% if n.status == 'ready' %}var(--green){% else %}var(--red){% endif %};">{{ n.status }}</span>
</td>
</tr>
{% else %}
<tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No nodes reported.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{% endfor %}
{% else %}
<div class="card empty-state">
No Swarm topology reported. A Steward host agent running on a Swarm
<strong>manager</strong> reports services, nodes and task placement here;
workers and hosts outside a swarm contribute only their own containers.
<div class="card" style="text-align:center;padding:3rem;">
<div style="color:var(--text-muted);font-size:0.9rem;">
No Swarm topology reported. A Steward host agent running on a Swarm
<strong>manager</strong> reports services, nodes, and task placement here —
workers and non-swarm hosts contribute only their local containers.
</div>
</div>
{% endif %}
{% endblock %}
plugins/docker/templates/docker/widget.html
+54 -67
View File
@@ -1,80 +1,67 @@
{# docker/widget.html — the dashboard Docker widget (#5252), in the widget-shape
order: a verdict line; the services and containers in trouble; what failed
in the last 24h, by name; then everything healthy, one line each, packed
into as many columns as the widget is wide. Resource use is the Busiest
Containers widget's, so it isn't repeated here. #}
{% from "_macros.html" import status_dot %}
{# docker/widget.html — dashboard widget: container status overview, by host #}
{% if total_count == 0 %}
<div class="w-verdict" style="color:var(--text-muted);">No containers reported yet.</div>
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No containers reported yet.</p>
{% else %}
{% set wrong = bad_services|length + bad_containers|length %}
<div class="w-verdict {{ 'tone-crit' if wrong else ('tone-warn' if failures else 'tone-ok') }}">
{%- if services_total %}{{ services_healthy }}/{{ services_total }} services healthy · {% endif -%}
{{ running_count }} other container{{ 's' if running_count != 1 }} running ·
{% if failures %}<a href="/plugins/docker/#failures" style="color:inherit;">{{ failures|length }} failed in 24h</a>{% else %}nothing failed in 24h{% endif -%}
<div style="display:flex;gap:1.25rem;margin-bottom:0.65rem;">
<div>
<span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--green);">{{ running_count }}</span>
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">running</span>
</div>
<div title="Distinct containers with a die or OOM event in the last 24h">
<span style="font-size:1.4rem;font-weight:700;line-height:1;color:{% if failed_24h %}var(--red){% else %}var(--text-muted){% endif %};">{{ failed_24h }}</span>
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">failed (24h)</span>
</div>
</div>
{% if wrong %}
<div class="w-section">Needs a look</div>
{% for s in bad_services %}
<div class="w-item">
{{ status_dot('degraded' if s.state == 'degraded' else 'down') }}
<span class="w-label" title="{{ s.name }}">{{ s.name }}
<span class="w-sub {{ 'tone-crit' if s.state != 'degraded' else 'tone-warn' }}">
{%- if s.crash_looping %}crash-looping: {{ s.failures_1h }} failed in the last hour{% else %}{{ s.running }} of {{ s.desired }} running{% endif %}
{%- if s.trouble_since %}, since {{ s.trouble_since|ago }}{% endif %}
{%- if s.last_error and (s.last_error.error or s.last_error.message) %} · {{ s.last_error.error or s.last_error.message }}{% endif -%}
</span>
</span>
<span class="w-meta" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
{# Swarm services — collapsed, each with its replicas' host chips (dashed = a
node with no Steward agent, count-only from placement). #}
{% if swarm_services %}
<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">Swarm services</div>
<div style="display:grid;gap:0.4rem;margin-bottom:0.4rem;">
{% for s in swarm_services %}
<div>
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
<span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
<span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
<span style="color:var(--text-muted);font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;">{{ s.running }}/{{ s.desired }}</span>
</div>
<div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">
{% for r in s.replicas %}
{% if r.ghost %}
<span title="no Steward agent on this node — count from swarm placement" style="font-size:0.68rem;color:var(--text-dim);background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }} ×{{ r.count }}</span>
{% else %}
<span style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }}</span>
{% endif %}
{% endfor %}
</div>
</div>
{% endfor %}
</div>
{% endfor %}
{% for c in bad_containers %}
<div class="w-item">
{{ status_dot('warn' if c.status == 'restarting' else 'down') }}
<span class="w-label" title="{{ c.name }}">
{%- if session.user_id %}<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>{% else %}{{ c.name }}{% endif %}
<span class="w-sub tone-crit">
{%- if c.health == 'unhealthy' %}unhealthy{% elif c.status == 'restarting' %}restarting{% else %}exited {{ c.exit_code }}{% if c.oom_killed %}, out of memory{% endif %}{% endif %} · {{ host_name.get(c.host_id, c.host_id) }}
</span>
</span>
</div>
{% endfor %}
{% endif %}
{% if failures %}
<div class="w-section">Failed in the last 24h</div>
{% for f in failures[:5] %}
<div class="w-item">
{{ status_dot('failed') }}
<span class="w-label" title="{{ f.name }}">{{ f.name }}
<span class="w-sub">{% if f.count > 1 %}{{ f.count }} times, last {% endif %}{{ f.at|ago }}{% if f.detail %} · {{ f.detail }}{% endif %} · {{ f.host }}</span>
</span>
</div>
{% endfor %}
{% if failures|length > 5 %}<div class="w-more">+{{ failures|length - 5 }} more · <a href="/plugins/docker/#failures">the full list</a></div>{% endif %}
{% for g in host_groups %}
{% if multi_host %}
<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">{{ g.host_name }}</div>
{% endif %}
{% if healthy %}
<div class="w-section">Healthy</div>
<div class="w-grid">
{% for h in healthy %}
<div class="w-item">
{% if h.kind == 'service' %}{% set s = h.s %}
{{ status_dot('up') }}
<span class="w-label" title="{{ s.name }} — on {{ s.replicas|map(attribute='host')|unique|join(', ') }}">{{ s.name }}
<span style="color:var(--text-dim);"> · {{ s.replicas|map(attribute='host')|unique|join(', ') }}</span></span>
<span class="w-meta" title="replicas running / wanted">{{ s.running }}/{{ s.desired }}</span>
{% else %}{% set c = h.c %}
{{ status_dot('up' if h.kind == 'container' else 'unknown') }}
<span class="w-label" title="{{ c.name }}">
{%- if session.user_id %}<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>{% else %}{{ c.name }}{% endif %}
<span style="color:var(--text-dim);"> · {{ host_name.get(c.host_id, c.host_id) }}</span></span>
{% if h.kind == 'stopped' %}<span class="w-meta">stopped</span>{% endif %}
<div style="display:grid;gap:2px;">
{% for c in g.containers %}
<div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
<span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;">
{{ c.name }}{% if c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);"> ●</span>{% elif c.health == 'healthy' %}<span title="healthy" style="color:var(--green);"> ●</span>{% endif %}{% if c.restart_count %}<span title="restarts" style="color:var(--orange);font-size:0.7rem;"> ⟳{{ c.restart_count }}</span>{% endif %}
</a>
{% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
<span title="last exit code" style="font-size:0.7rem;color:var(--red);flex-shrink:0;">exit {{ c.exit_code }}</span>
{% endif %}
{% if c.cpu_pct is not none %}
<span style="font-size:0.72rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
{{ "%.1f" | format(c.cpu_pct) }}%
</span>
{% endif %}
</div>
{% endfor %}
</div>
{% if healthy_more %}<div class="w-more">+{{ healthy_more }} more healthy · <a href="/plugins/docker/">all of Docker</a></div>{% endif %}
{% endif %}
{% endfor %}
{% endif %}
plugins/docker/templates/docker/widget_resources.html
+23 -45
View File
@@ -1,51 +1,29 @@
{# docker/widget_resources.html — the dashboard Busiest Containers widget
(#5253, #5214): a verdict naming the busiest, then the top containers by CPU
(averaged over the last few minutes, so the order holds still). Each figure
carries its unit, beside bars of the last few hours. A row's bars are scaled
to its own peak, but never below a floor (1 core, 1 GiB by default), so a
container idling at 0.05 cores reads flat and a spike means real load. #}
{% macro cores(v) -%}{{ "%.2f"|format(v) if v < 1 else "%.1f"|format(v) }}{%- endmacro %}
{% macro bars(b, peak_text, roof_text) -%}
<span class="w-bars w-wide-only" aria-hidden="true"
title="{% if b.peak is none %}No samples in the last {{ hours }}h{% else %}Peak {{ peak_text }} in the last {{ hours }}h; bars reach the top at {{ roof_text }}{% endif %}">
{%- for h in b.bars %}<i{% if h is none %} class="gap"{% else %} style="height:{{ [h, 4]|max }}%"{% endif %}></i>{% endfor -%}
</span>
{%- endmacro %}
{# docker/widget_resources.html — dashboard widget: the busiest running containers
by CPU, each with labeled CPU + memory utilisation bars. #}
{% if not rows %}
<div class="w-verdict" style="color:var(--text-muted);">No running containers.</div>
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No running containers.</div>
{% else %}
{% set top = rows[0] %}
<div class="w-verdict">
Busiest: <strong>{{ top.label }}</strong> on {{ top.host_name }},
{% if top.usage.cores is not none %}{{ cores(top.usage.cores) }}
{%- if top.usage.cpus %} of {{ top.usage.cpus }} cores{% else %} cores{% endif %}{% else %}no CPU reading yet{% endif %}.
{% macro bar(label, pct, warn, crit) %}
<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;">
<span style="font-size:0.62rem;color:var(--text-dim);width:1.9rem;flex-shrink:0;letter-spacing:0.03em;">{{ label }}</span>
<div style="flex:1;height:6px;background:var(--bg-elevated);border-radius:3px;overflow:hidden;">
<div style="height:100%;border-radius:3px;width:{{ [pct, 100] | min }}%;
background:{% if pct > crit %}var(--red){% elif pct > warn %}var(--orange){% else %}var(--accent){% endif %};
transition:width 0.4s;"></div>
</div>
<span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;width:3rem;text-align:right;flex-shrink:0;">{{ "%.1f" | format(pct) }}%</span>
</div>
<table class="w-table">
<colgroup><col><col class="w-wide-only" style="width:7.5rem"><col class="w-barcol"><col class="w-barcol"></colgroup>
<thead><tr><th>Container</th><th class="w-wide-only w-text">Host</th>
<th title="Cores, averaged over the last {{ minutes }} minutes; bars: the last {{ hours }}h">CPU<span class="w-wide-only"> · last {{ hours }}h</span></th>
<th title="In use now; bars: the last {{ hours }}h">Memory<span class="w-wide-only"> · last {{ hours }}h</span></th></tr></thead>
<tbody>
{% endmacro %}
<div style="display:grid;gap:0.6rem;">
{% for r in rows %}
{% set u = r.usage %}
<tr>
<td class="w-name" title="{{ r.c.name }}">
{%- if session.user_id %}<a href="/plugins/docker/container/{{ r.c.host_id }}/{{ r.c.name }}">{{ r.label }}</a>{% else %}{{ r.label }}{% endif -%}
</td>
<td class="w-wide-only w-text" title="{{ r.host_name }}">{{ r.host_name }}</td>
<td class="tone-{{ (u.cpu_grade or status_class(u.cpu_share, 'cpu')) if u.cpus and u.cores is not none else 'dim' }}"
{% if u.cpus and u.cores is not none %}title="{{ '%.1f'|format(u.cpu_share) }}% of {{ r.host_name }}'s {{ u.cpus }} cores{% if u.cpu_usual %}; {{ u.cpu_usual }}{% endif %}"{% endif %}>
{{- bars(r.cpu_bars, (cores(r.cpu_bars.peak) ~ " cores") if r.cpu_bars.peak is not none else "", cores(r.cpu_bars.roof) ~ " cores") -}}
<span class="w-fig">{% if u.cores is none %}—{% else %}{{ cores(u.cores) }} cores{% endif %}</span>
</td>
<td class="tone-{{ (u.mem_grade or status_class(u.mem_pct, 'mem')) if u.mem_of and u.mem_bytes is not none else 'dim' }}"
{% if u.mem_pct is not none and u.mem_of and u.mem_bytes is not none %}title="{{ '%.1f'|format(u.mem_pct) }}% of {{ u.mem_of }}{% if u.mem_usual %}, {{ u.mem_usual }}{% endif %}"{% endif %}>
{{- bars(r.mem_bars, r.mem_bars.peak|bytes if r.mem_bars.peak is not none else "", r.mem_bars.roof|bytes) -}}
<span class="w-fig">{% if u.mem_bytes is none %}—{% else %}{{ u.mem_bytes|bytes }}{% endif %}</span>
</td>
</tr>
{% set c = r.c %}
<div>
<div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;">
{{ c.name }}{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
</div>
{% if c.cpu_pct is not none %}{{ bar("CPU", c.cpu_pct, 50, 80) }}{% endif %}
{% if c.mem_pct is not none %}{{ bar("MEM", c.mem_pct, 70, 90) }}{% endif %}
</div>
{% endfor %}
</tbody>
</table>
<div class="w-note w-wide-only">Bars: {{ (hours * 60 // 24) }}-minute slices, oldest left; each row up to its own peak, at least {{ cores(cpu_floor) }} core{{ 's' if cpu_floor != 1 }} and {{ mem_floor|bytes }}.</div>
</div>
{% endif %}
plugins/host_agent/__init__.py
-32
View File
@@ -11,14 +11,6 @@ _app: "Quart | None" = None
def setup(app: "Quart") -> None:
global _app
_app = app
# A host's CPU, memory and load per core learn their normal (#5742), each
# with its floor from Settings → Thresholds. Parts don't: a host's CPU has
# a normal, each of its cores doesn't. Disks stay on their fixed cutoffs:
# a nearly full disk is a failure whatever its history (System 182).
from steward.core.norms import floor_setting, learn_normal
learn_normal("host_agent", "cpu_pct", floor_setting(app, "host_cpu_pct"), parts=False)
learn_normal("host_agent", "mem_used_pct", floor_setting(app, "host_mem_pct"), parts=False)
learn_normal("host_agent", "load_pct", floor_setting(app, "host_load_pct"), parts=False)
def get_scheduled_tasks() -> list:
@@ -29,27 +21,3 @@ def get_scheduled_tasks() -> list:
def get_blueprint():
from .routes import host_agent_bp
return host_agent_bp
def register_mcp_tools(register) -> None:
from .mcp_tools import register_tools
register_tools(register)
async def mcp_overview(db) -> list[dict]:
from .mcp_tools import mcp_overview as findings
return await findings(db)
def get_alert_metrics() -> dict[str, list[str]]:
"""Host metrics ingest records for alert rules (resource = the host name;
disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen
comes from the scheduler, not the agent: "agent offline" is > 180.
clock_off_secs is how far the host's clock is from Steward's, either way
(agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a %
of the host's cores."""
return {"host_agent": [
"cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
"disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m", "load_pct",
"uptime_secs", "seconds_since_seen", "clock_off_secs",
]}
plugins/host_agent/agent.py
+59 -473
View File
@@ -17,10 +17,10 @@ import time
import urllib.error
import urllib.request
from collections import deque
from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
AGENT_VERSION = "1.12.0"
AGENT_VERSION = "1.6.0"
# Default path to the local Docker Engine socket. Overridable via the
# `docker_socket` config key; collection is silently skipped if it's absent or
@@ -34,10 +34,7 @@ class ConfigError(Exception):
REQUIRED_KEYS = ("url", "token")
INT_KEYS = ("interval_seconds",)
LIST_KEYS = ("mounts", "docker_log_exclude")
# Truthy strings for bool keys; anything else (incl. empty) is False.
BOOL_KEYS = ("docker_logs_enabled",)
_BOOL_TRUE = ("1", "true", "yes", "on")
LIST_KEYS = ("mounts",)
def read_config(path: str) -> dict:
@@ -61,8 +58,6 @@ def read_config(path: str) -> dict:
raise ConfigError(f"{path}:{lineno}: {key} must be int")
elif key in LIST_KEYS:
cfg[key] = [v.strip() for v in value.split(",") if v.strip()]
elif key in BOOL_KEYS:
cfg[key] = value.lower() in _BOOL_TRUE
else:
cfg[key] = value
except FileNotFoundError:
@@ -74,10 +69,6 @@ def read_config(path: str) -> dict:
cfg.setdefault("interval_seconds", 30)
cfg.setdefault("docker_socket", DEFAULT_DOCKER_SOCKET)
# Container-log collection is on by default (operator preference); a host can
# opt out with `docker_logs_enabled = false` or thin it with docker_log_exclude.
cfg.setdefault("docker_logs_enabled", True)
cfg.setdefault("docker_log_exclude", [])
return cfg
@@ -91,10 +82,6 @@ OS_RELEASE_PATH = "/etc/os-release"
NETDEV_PATH = "/proc/net/dev"
DISKSTATS_PATH = "/proc/diskstats"
HWMON_DIR = "/sys/class/hwmon"
# Where the kernel lists virtual network devices (bridges, veth, docker0,
# vxlan, tun, VLANs, bonds) and the block devices (a disk has a `device`).
SYS_VIRTUAL_NET_DIR = "/sys/devices/virtual/net"
SYS_BLOCK_DIR = "/sys/block"
PSI_DIR = "/proc/pressure"
# Partition names to drop from disk I/O (we report whole-disk throughput only).
@@ -263,34 +250,10 @@ def default_mounts() -> list[str]:
return mounts or ["/"]
def _physical_only(found: dict, is_virtual) -> dict:
"""The entries the kernel says are physical, so a host total counts each
byte once (a bridge's or an LVM volume's traffic is also counted on the
NIC or disk beneath it). When /sys calls every one of them virtual (an
agent inside LXC, where eth0 is a veth), they are all kept: a host
reporting no traffic at all would be wrong, not filtered."""
physical = {k: v for k, v in found.items() if not is_virtual(k)}
return physical or found
def _virtual_net(name: str) -> bool:
"""Whether the kernel lists this interface as virtual; False if /sys can't say."""
return os.path.isdir(SYS_VIRTUAL_NET_DIR) and os.path.exists(
os.path.join(SYS_VIRTUAL_NET_DIR, name))
def _virtual_disk(name: str) -> bool:
"""Whether this block device has no disk behind it (dm, md, zram, nbd);
False if /sys can't say."""
return os.path.isdir(SYS_BLOCK_DIR) and not os.path.exists(
os.path.join(SYS_BLOCK_DIR, name, "device"))
def collect_net_raw() -> dict:
"""Return {iface: (rx_bytes, tx_bytes)} cumulative counters from /proc/net/dev.
Physical interfaces only (see _physical_only); loopback and veth* are
skipped by name too, for a host whose /sys can't be read. Counters are
Skips loopback and veth* (container) interfaces to cut noise. Counters are
monotonic; per-second rates are derived from deltas in build_sample().
"""
out: dict = {}
@@ -312,15 +275,14 @@ def collect_net_raw() -> dict:
out[name] = (int(f[0]), int(f[8])) # rx_bytes, tx_bytes
except ValueError:
continue
return _physical_only(out, _virtual_net)
return out
def collect_diskio_raw() -> dict:
"""Return {device: (read_bytes, write_bytes)} cumulative from /proc/diskstats.
Physical whole disks only (see _physical_only): loop/ram/sr/fd and
partitions are skipped by name too. Sectors are 512 bytes. Counters are
monotonic; rates derived from deltas.
Whole disks only — loop/ram/sr/fd and partitions are skipped. Sectors are
512 bytes. Counters are monotonic; rates derived from deltas.
"""
out: dict = {}
try:
@@ -338,7 +300,7 @@ def collect_diskio_raw() -> dict:
out[name] = (int(f[5]) * 512, int(f[9]) * 512) # sectors_read, sectors_written
except ValueError:
continue
return _physical_only(out, _virtual_disk)
return out
def collect_temps() -> list:
@@ -440,24 +402,6 @@ def _rates(cur: dict, prev: dict, dt: float) -> dict:
# when that header is present rather than assume Content-Length.
DOCKER_API_TIMEOUT = 5.0
# All of one sample's per-container Docker work (inspect + stats) must finish
# inside this; past it the sample says Docker did not answer in time rather
# than holding up the host's own metrics (#5154).
DOCKER_DEADLINE_SECONDS = 20.0
# /system/df walks every image, container and volume: asked this often, not
# every sample. Steward keeps the last answer between reports.
DOCKER_DF_INTERVAL_SECONDS = 600.0
# Container-log collection (m79). Logs ride in the same push as metrics, so one
# interval's batch is capped to keep a chatty container from bloating a POST (and
# the backoff ring): once the cap is hit a marker line is emitted and collection
# stops — the deferred lines come on the next interval. On a container's first
# interval we seed from a short tail, then switch to an incremental since-cursor
# kept per container in the agent's rate-state.
DOCKER_LOG_BATCH_MAX_BYTES = 262144 # 256 KiB of log text per push, all containers
DOCKER_LOG_FIRST_TAIL = 50 # lines seeded on a container's first interval
DOCKER_LOG_FINAL_TAIL = 200 # lines read once from a container found already crashed
DOCKER_LOG_FINAL_WINDOW_SECONDS = 600 # ... when it stopped this recently
def _dechunk(body: bytes) -> bytes:
@@ -601,18 +545,6 @@ def _docker_inspect_fields(insp: dict):
return health, restart_count, exit_code, oom_killed
def _docker_started_at(insp: dict, key: str = "StartedAt"):
"""State.StartedAt as ISO-8601, or None for a container never started
(Docker's zero time) or an inspect that failed. When the container last
started, not when it was created: a restarted container's uptime counts
from its restart. key="FinishedAt" reads when it last stopped."""
raw = (((insp or {}).get("State") or {}).get(key) or "")
if not raw or raw.startswith("0001-"):
return None
ts = _parse_log_ts(raw)
return ts.isoformat() if ts else None
def _docker_net_io(stats: dict):
"""Return cumulative (net_rx, net_tx, blk_read, blk_write) bytes from stats.
@@ -646,7 +578,7 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
# Inspect every container (incl. stopped) — exit codes + restart counts only
# live here, and stopped containers are exactly where exit_code matters.
health = exit_code = insp = None
health = exit_code = None
restart_count = 0
oom_killed = False
try:
@@ -666,10 +598,11 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
except (OSError, ValueError):
pass
started_at = _docker_started_at(insp)
# When it last stopped (a running container's is its previous stop, or
# None): dates a die event, and tells the log collector a stop is new.
finished_at = None if state == "running" else _docker_started_at(insp, "FinishedAt")
created = c.get("Created")
started_at = (
datetime.fromtimestamp(created, tz=timezone.utc).isoformat()
if isinstance(created, (int, float)) else None
)
record = {
"name": name,
"container_id": cid[:12],
@@ -681,7 +614,6 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
"mem_pct": mem_pct,
"ports": _docker_ports(c.get("Ports", [])),
"started_at": started_at,
"finished_at": finished_at,
"health": health,
"restart_count": restart_count,
"exit_code": exit_code,
@@ -695,276 +627,27 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
return record
def _docker_failure(socket_path: str, exc: Exception) -> tuple:
"""(status, detail) for a failed Docker call: "unavailable" when there is no
Docker here to ask, "error" when there is one and the call went wrong."""
if isinstance(exc, FileNotFoundError):
return "unavailable", "no Docker socket at " + socket_path
if isinstance(exc, ConnectionRefusedError):
return "unavailable", "Docker is not accepting connections on " + socket_path
if isinstance(exc, PermissionError):
return "error", "permission denied on " + socket_path
if isinstance(exc, socket.timeout):
return "error", "Docker did not answer within %.0f s" % DOCKER_API_TIMEOUT
return "error", (str(exc) or type(exc).__name__)[:200]
def collect_docker(socket_path: str) -> list:
"""Per-container state from the local Docker socket, or [] if unavailable.
def collect_docker(socket_path: str) -> tuple:
"""(status, detail, containers) from the local Docker socket.
status is "ok" when Docker answered (containers may then be [] on a host
with none, which tells Steward to clear its list), "unavailable" when there
is no Docker here (no socket, or nothing listening), and "error" when Docker
is here but the listing failed. detail says why, for the last two. A
Docker-less host still needs no configuration: it reports "unavailable".
Absent socket / permission denied / non-Docker endpoint all yield [] — the
agent silently reports no containers rather than erroring, so a Docker-less
host needs no configuration.
"""
try:
containers = _docker_request(socket_path, "/containers/json?all=true")
except (OSError, ValueError) as exc:
return _docker_failure(socket_path, exc) + ([],)
if not isinstance(containers, list):
return "error", "unexpected answer from /containers/json", []
if not containers:
return "ok", None, []
except (OSError, ValueError):
return []
if not isinstance(containers, list) or not containers:
return []
# Each container needs an inspect (+ a stats read if running), and the stats
# call blocks ~1s while the daemon computes the cpu delta. Done serially that
# stretches the whole sample on a busy host, so fan the per-container work out
# over a small bounded thread pool (I/O-bound → threads are enough).
# A deadline on the whole fan-out: one wedged container must not stall the
# sample. Unfinished work is reported as Docker not answering in time, not
# as a partial list: a container left out would read as removed, and one
# without its inspect as restarted or changed health.
ex = ThreadPoolExecutor(max_workers=min(8, len(containers)))
try:
futures = [ex.submit(_collect_one_container, socket_path, c) for c in containers]
done, pending = wait_futures(futures, timeout=DOCKER_DEADLINE_SECONDS)
finally:
try:
ex.shutdown(wait=False, cancel_futures=True)
except TypeError: # Python < 3.9: no cancel_futures
ex.shutdown(wait=False)
if pending:
return ("error", "%d of %d containers not read within %g s"
% (len(pending), len(containers), DOCKER_DEADLINE_SECONDS), [])
return "ok", None, [f.result() for f in futures]
# ─── container logs (m79) ─────────────────────────────────────────────────────
#
# The logs endpoint returns a raw multiplexed byte stream, not JSON, so it needs
# a sibling of _docker_request that hands back bytes. Non-TTY containers frame
# stdout/stderr with an 8-byte header (stream byte + 4-byte big-endian length);
# TTY containers emit unframed bytes. We request timestamps=1, so every line is
# prefixed with an RFC3339Nano timestamp we parse for the since-cursor.
_LOG_STREAM_NAMES = {0: "stdout", 1: "stdout", 2: "stderr"}
def _docker_request_raw(socket_path: str, path: str,
timeout: float = DOCKER_API_TIMEOUT) -> bytes:
"""GET `path` from the Docker API over the Unix socket; return the raw body.
Sibling of `_docker_request` for non-JSON endpoints (container logs). Reuses
the connect / send / de-chunk scaffolding; raises OSError on any transport
problem or non-2xx status so callers can silent-skip. A 200 with an empty
body (no new lines since the cursor) returns b"".
"""
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(timeout)
try:
sock.connect(socket_path)
req = (
"GET " + path + " HTTP/1.1\r\n"
"Host: docker\r\n"
"Accept: application/octet-stream\r\n"
"Connection: close\r\n"
"\r\n"
)
sock.sendall(req.encode("ascii"))
chunks = []
while True:
buf = sock.recv(65536)
if not buf:
break
chunks.append(buf)
finally:
sock.close()
raw = b"".join(chunks)
head, _, body = raw.partition(b"\r\n\r\n")
header_text = head.decode("latin-1")
status_line = header_text.split("\r\n", 1)[0]
parts = status_line.split(None, 2)
status = int(parts[1]) if len(parts) >= 2 and parts[1].isdigit() else 0
if not (200 <= status < 300):
raise OSError(f"docker API {path} returned {status}")
if "transfer-encoding: chunked" in header_text.lower():
body = _dechunk(body)
return body
def _demux_docker_logs(raw: bytes) -> list:
"""Split a Docker logs stream into [(stream_name, payload_bytes), …].
Non-TTY containers multiplex with an 8-byte frame header (stream byte in
{0,1,2} + 4-byte big-endian length). The stream is treated as framed only if
the whole buffer parses as clean back-to-back frames; anything else (a TTY
container's raw stream, or a malformed header) falls back to a single stdout
blob so no bytes are lost.
"""
if not raw:
return []
frames = []
i, n = 0, len(raw)
while i + 8 <= n:
stream = raw[i]
if stream > 2 or raw[i + 1] or raw[i + 2] or raw[i + 3]:
break # not a valid frame header → not framed
size = int.from_bytes(raw[i + 4:i + 8], "big")
if i + 8 + size > n:
break # frame overruns the buffer → not framed
frames.append((_LOG_STREAM_NAMES.get(stream, "stdout"),
raw[i + 8:i + 8 + size]))
i += 8 + size
if frames and i == n:
return frames
return [("stdout", raw)]
def _parse_log_ts(token: str):
"""Parse a Docker RFC3339Nano timestamp token into a datetime, or None.
Normalises `Z` → `+00:00` and trims Docker's nanosecond precision to the
microseconds datetime.fromisoformat accepts (stdlib, Python 3.8+ safe).
"""
t = token.strip()
if not t:
return None
if t.endswith("Z"):
t = t[:-1] + "+00:00"
m = re.match(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(\.\d+)?(.*)$", t)
if m:
frac = m.group(2) or ""
if len(frac) > 7: # "." + up to 6 fractional digits
frac = frac[:7]
t = m.group(1) + frac + m.group(3)
try:
return datetime.fromisoformat(t)
except ValueError:
return None
def _parse_container_logs(raw: bytes) -> list:
"""Demux + line-split a logs stream → [(dt|None, stream, line), …] in order.
Each line is timestamp-prefixed (we request timestamps=1); an unparseable
prefix yields dt=None with the full raw line kept.
"""
out = []
for stream, payload in _demux_docker_logs(raw):
text = payload.decode("utf-8", "replace")
for raw_line in text.split("\n"):
if not raw_line:
continue
token, _, msg = raw_line.partition(" ")
dt = _parse_log_ts(token)
if dt is not None:
out.append((dt, stream, msg))
else:
out.append((None, stream, raw_line))
return out
def collect_docker_logs(socket_path: str, containers: list, state: dict,
exclude=None,
max_bytes: int = DOCKER_LOG_BATCH_MAX_BYTES,
now: float | None = None) -> list:
"""New container-log lines since the last interval.
Keeps a per-container since-cursor in `state["docker_log_cursors"]`
(name → unix ts) so each interval fetches only what's new; a container's
first interval seeds from a short tail. A running container is read every
interval. A stopped one is read once per stop (`finished_at`, remembered in
`state["docker_log_finals"]`), so the lines it wrote on its way down reach
Steward (#5049): since its cursor if it was being read, or, for one first
seen already stopped with a non-zero exit in the last
DOCKER_LOG_FINAL_WINDOW_SECONDS, its last DOCKER_LOG_FINAL_TAIL lines. That
is how a task that crashes between two samples is explained by its own
output. Both maps forget a container once it is no longer listed.
The whole batch is capped at `max_bytes` of log text — once exceeded a
truncation marker is appended and collection stops (logs must never balloon
a push). `exclude` container names are skipped entirely (a per-host
bandwidth opt-out). Best-effort: a per-container transport error just
contributes nothing this interval.
Returns [{"container", "stream", "ts", "line"}, …].
"""
exclude_set = set(exclude or ())
cursors = state.setdefault("docker_log_cursors", {})
finals = state.setdefault("docker_log_finals", {})
now = time.time() if now is None else now
listed = {c.get("name"): c for c in (containers or []) if c.get("name")}
# Forget containers no longer listed so state can't grow without bound
# over the agent's lifetime.
for book in (cursors, finals):
for gone in [k for k in book if k not in listed]:
del book[gone]
out: list = []
total = 0
truncated = False
for name, c in listed.items():
if name in exclude_set:
continue
prev = cursors.get(name)
base = f"/containers/{name}/logs?stdout=1&stderr=1&timestamps=1"
if c.get("status") == "running":
# since accepts a fractional Unix ts; we still dedup the boundary line.
path = (f"{base}&tail={DOCKER_LOG_FIRST_TAIL}" if prev is None
else f"{base}&since={prev:.9f}")
else:
stopped = c.get("finished_at")
if not stopped or finals.get(name) == stopped:
continue
if prev is not None:
path = f"{base}&since={prev:.9f}"
else:
at = _parse_log_ts(stopped)
if (c.get("exit_code") in (None, 0) or at is None
or now - at.timestamp() > DOCKER_LOG_FINAL_WINDOW_SECONDS):
continue
path = f"{base}&tail={DOCKER_LOG_FINAL_TAIL}"
finals[name] = stopped
try:
raw = _docker_request_raw(socket_path, path)
except (OSError, ValueError):
continue
newest = prev
for dt, stream, line in _parse_container_logs(raw):
ts_unix = dt.timestamp() if dt is not None else None
if prev is not None and ts_unix is not None and ts_unix <= prev:
continue # boundary line already sent last interval
if total >= max_bytes:
truncated = True
break
out.append({"container": name, "stream": stream,
"ts": dt.isoformat() if dt is not None else None,
"line": line})
total += len(line)
if ts_unix is not None and (newest is None or ts_unix > newest):
newest = ts_unix
if newest is not None:
cursors[name] = newest
if truncated:
break
if truncated:
out.append({"container": "_steward", "stream": "stderr",
"ts": datetime.now(timezone.utc).isoformat(),
"line": (f"[steward] log batch truncated at {max_bytes} bytes; "
"remaining lines deferred to the next interval")})
return out
workers = min(8, len(containers))
with ThreadPoolExecutor(max_workers=workers) as ex:
return list(ex.map(lambda c: _collect_one_container(socket_path, c), containers))
# ─── swarm (manager-only) ────────────────────────────────────────────────────
@@ -981,46 +664,14 @@ def _swarm_is_manager(info: dict) -> bool:
return bool((info or {}).get("Swarm", {}).get("ControlAvailable"))
# Newest tasks kept per service. Swarm retains about 5 per slot by default, so
# this covers a replicated service's recent history and a crash loop's deaths
# without shipping a global service's whole fleet every sample.
SWARM_TASKS_PER_SERVICE = 10
_TASK_TEXT_CHARS = 300
def _swarm_task(t: dict) -> dict:
"""One task as Steward stores it: where it ran, what Swarm wants of it,
what it is doing, and why it stopped (Err/Message, exit code)."""
status = t.get("Status") or {}
cstatus = status.get("ContainerStatus") or {}
exit_code = cstatus.get("ExitCode")
return {
"task_id": t.get("ID", "") or "",
"slot": t.get("Slot"),
"node_id": t.get("NodeID", "") or "",
"desired_state": (t.get("DesiredState") or "").lower(),
"state": (status.get("State") or "").lower(),
"message": (status.get("Message") or "")[:_TASK_TEXT_CHARS],
"error": (status.get("Err") or "")[:_TASK_TEXT_CHARS],
"exit_code": exit_code if isinstance(exit_code, int) else None,
"at": status.get("Timestamp") or t.get("UpdatedAt") or "",
}
def _swarm_services(services: list, tasks: list) -> list:
"""Roll desired-vs-running replicas per service up from the task list.
Replica health isn't on the service object — it's derived by counting tasks:
`running` = tasks currently in state running; `desired` = the service's
configured replica count (replicated) or the count of tasks the scheduler
wants running (global, which has no fixed number). Each service also
carries its newest tasks (`tasks`), so Steward can tell a task Swarm
replaced from one that crashed, and say why the last one failed.
wants running (global, which has no fixed number).
"""
tasks_by_svc: dict = {}
for t in (tasks or []):
if t.get("ServiceID"):
tasks_by_svc.setdefault(t["ServiceID"], []).append(_swarm_task(t))
running_by_svc: dict = {}
desired_by_svc: dict = {} # only used for global-mode services
placement: dict = {} # svc_id -> {node_id: running_count}
@@ -1060,8 +711,6 @@ def _swarm_services(services: list, tasks: list) -> list:
# sees every task, even ones whose containers live on other hosts).
"placement": [{"node_id": n, "running": cnt}
for n, cnt in sorted(placement.get(sid, {}).items())],
"tasks": sorted(tasks_by_svc.get(sid, []), key=lambda t: t["at"],
reverse=True)[:SWARM_TASKS_PER_SERVICE],
})
return out
@@ -1085,28 +734,26 @@ def _swarm_nodes(nodes: list) -> list:
return out
def collect_swarm(socket_path: str) -> tuple:
"""(is_manager, topology) from the local Docker socket.
def collect_swarm(socket_path: str):
"""Swarm topology from a manager node, or None on workers / non-swarm hosts.
Self-detects manager via `/info` (one cheap call); only then queries the
manager-only endpoints. is_manager is None when /info itself failed, so a
transient error is never mistaken for a node that stopped being a manager;
False tells Steward to drop the swarm rows this host reported as one.
topology is None unless this is a manager and every call answered.
manager-only endpoints. Any transport failure degrades to None, same silent
contract as collect_docker — a non-swarm host adds nothing to the payload.
"""
try:
info = _docker_request(socket_path, "/info")
except (OSError, ValueError):
return None, None
return None
if not _swarm_is_manager(info):
return False, None
return None
try:
services = _docker_request(socket_path, "/services")
tasks = _docker_request(socket_path, "/tasks")
nodes = _docker_request(socket_path, "/nodes")
except (OSError, ValueError):
return True, None
return True, {
return None
return {
"services": _swarm_services(
services if isinstance(services, list) else [],
tasks if isinstance(tasks, list) else []),
@@ -1205,17 +852,13 @@ class RingBuffer:
def build_sample(mounts: list[str], state: dict,
docker_socket: str = DEFAULT_DOCKER_SOCKET, *,
docker_logs_enabled: bool = True,
docker_log_exclude=None) -> dict:
docker_socket: str = DEFAULT_DOCKER_SOCKET) -> dict:
"""Collect one full sample. Partial samples allowed if a collector fails.
`state` carries the previous network/disk counters + monotonic timestamp so
throughput rates can be derived from deltas (and the per-container log
since-cursors); it is mutated in place. `docker_socket` is probed
best-effort — the `docker` key is omitted entirely when no containers are
found, so non-Docker hosts add nothing to the payload. `docker_logs_enabled`
/ `docker_log_exclude` gate incremental container-log collection.
throughput rates can be derived from deltas; it is mutated in place.
`docker_socket` is probed best-effort — the `docker` key is omitted entirely
when no containers are found, so non-Docker hosts add nothing to the payload.
"""
sample: dict = {"ts": datetime.now(timezone.utc).isoformat()}
try:
@@ -1270,33 +913,26 @@ def build_sample(mounts: list[str], state: dict,
]
state["net"], state["disk"], state["mono"] = net_raw, disk_raw, now_mono
# Docker's status rides every sample, so Steward can tell "no containers"
# (status ok, docker: []) from "couldn't ask" (unavailable / error).
try:
docker_status, docker_detail, docker = collect_docker(docker_socket)
except Exception as exc:
docker_status, docker_detail, docker = "error", (str(exc) or type(exc).__name__)[:200], []
sample["docker_status"] = docker_status
if docker_detail:
sample["docker_detail"] = docker_detail
if docker_status == "ok":
docker = collect_docker(docker_socket)
except Exception:
docker = []
if docker:
sample["docker"] = docker
# Swarm topology is only emitted by managers. A node whose /info says it
# is not a manager says so, so Steward can expire what it once reported.
# Swarm topology is only emitted by managers (collect_swarm returns None
# everywhere else), so the key is absent on workers / non-swarm hosts.
try:
is_manager, swarm = collect_swarm(docker_socket)
swarm = collect_swarm(docker_socket)
except Exception:
is_manager, swarm = None, None
swarm = None
if swarm is not None:
sample["swarm"] = swarm
elif is_manager is False:
sample["swarm_manager"] = False
# Image/disk usage — host-level docker fact (one /system/df call), only
# when Docker answered.
if docker_status == "ok" and now_mono - state.get("df_mono", float("-inf")) >= DOCKER_DF_INTERVAL_SECONDS:
state["df_mono"] = now_mono
# Image/disk usage — host-level docker fact (one /system/df call), omitted on
# Docker-less hosts. Only meaningful when there's docker here at all, so skip
# the call entirely when collect_docker found nothing.
if docker:
try:
disk = collect_disk_usage(docker_socket)
except Exception:
@@ -1304,29 +940,14 @@ def build_sample(mounts: list[str], state: dict,
if disk is not None:
sample["docker_disk"] = disk
# Container logs (m79): incremental per-container tail folded into the same
# push as metrics. Skipped when there are no containers or logging is off;
# the key is omitted when nothing new arrived this interval.
if docker and docker_logs_enabled:
try:
logs = collect_docker_logs(docker_socket, docker, state,
exclude=docker_log_exclude)
except Exception:
logs = []
if logs:
sample["docker_logs"] = logs
return sample
def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict:
# sent_at is this host's clock at the moment of sending: Steward compares
# it with its own on arrival to say how far apart the two clocks are.
return {
"agent_version": AGENT_VERSION,
"hostname": hostname,
"metadata": metadata,
"sent_at": datetime.now(timezone.utc).isoformat(),
"samples": samples,
}
@@ -1334,32 +955,7 @@ def build_payload(samples: list[dict], hostname: str, metadata: dict) -> dict:
# ─── POST + backoff ──────────────────────────────────────────────────────────
# Below Steward's 180 s staleness window (host_agent stale_after_seconds): once
# Steward is reachable again a host reports within a minute, instead of reading
# as stale for up to five (#5154; lesson: a check-in inherits its call's backoff).
BACKOFF_CAP = 60
# Samples in one POST: the server writes at most this many (routes.py
# MAX_SAMPLES_PER_INGEST), so a backlog drains a few per post, oldest first,
# rather than in one post too big to write before the timeout.
MAX_SAMPLES_PER_POST = 5
def _drop_logs(sample: dict) -> dict:
"""Strip container logs from a sample before it's buffered for retry.
Logs are the one payload we never carry across a backoff — they'd bloat the
ring buffer and go stale — while metrics are kept so an outage doesn't lose
them. The since-cursor already advanced when the logs were collected, so the
dropped lines are simply not re-sent (accepted loss during an outage).
"""
sample.pop("docker_logs", None)
return sample
def split_backlog(buffered: list, limit: int = MAX_SAMPLES_PER_POST) -> tuple:
"""(the buffered samples to send with this one, the ones held for later):
the oldest limit-1, leaving room for the new sample."""
return buffered[:limit - 1], buffered[limit - 1:]
BACKOFF_CAP = 300
def next_backoff(current: int) -> int:
@@ -1426,8 +1022,6 @@ def main_loop(conf_path: str) -> int:
hostname = cfg.get("hostname") or socket.gethostname()
mounts = cfg.get("mounts") or default_mounts()
docker_socket = cfg.get("docker_socket") or DEFAULT_DOCKER_SOCKET
docker_logs_enabled = cfg.get("docker_logs_enabled", True)
docker_log_exclude = cfg.get("docker_log_exclude") or []
buffer = RingBuffer(maxlen=20)
backoff = 0
# Carries previous net/disk counters + monotonic ts for rate computation.
@@ -1442,32 +1036,24 @@ def main_loop(conf_path: str) -> int:
cfg = read_config(conf_path)
mounts = cfg.get("mounts") or default_mounts()
docker_socket = cfg.get("docker_socket") or DEFAULT_DOCKER_SOCKET
docker_logs_enabled = cfg.get("docker_logs_enabled", True)
docker_log_exclude = cfg.get("docker_log_exclude") or []
metadata = collect_metadata() # refresh host_ip/distro on reload
_log("INFO", "config reloaded")
except ConfigError as e:
_log("ERROR", f"reload failed: {e}")
_reload_requested = False
sample = build_sample(mounts, rate_state, docker_socket,
docker_logs_enabled=docker_logs_enabled,
docker_log_exclude=docker_log_exclude)
sample = build_sample(mounts, rate_state, docker_socket)
buffered = buffer.drain()
send, held = split_backlog(buffered)
payload = build_payload(
samples=send + [sample],
samples=buffered + [sample],
hostname=hostname,
metadata=metadata,
)
ok, status = post_payload(cfg["url"], cfg["token"], payload)
if ok:
for s in held:
buffer.push(s)
if send:
_log("INFO", f"flushed {len(send)} buffered samples"
+ (f", {len(held)} still held" if held else ""))
if buffered:
_log("INFO", f"flushed {len(buffered)} buffered samples")
backoff = 0
sleep_for = cfg["interval_seconds"]
else:
@@ -1475,12 +1061,12 @@ def main_loop(conf_path: str) -> int:
_log("ERROR", "server rejected payload (400) — dropping sample")
elif status == 401:
_log("ERROR", "token rejected (401) — check config + UI")
buffer.push(_drop_logs(sample))
buffer.push(sample)
else:
_log("WARN", f"POST failed (status={status}); buffering")
for s in buffered:
buffer.push(_drop_logs(s))
buffer.push(_drop_logs(sample))
buffer.push(s)
buffer.push(sample)
backoff = next_backoff(backoff)
sleep_for = backoff
plugins/host_agent/mcp_tools.py
-81
View File
@@ -1,81 +0,0 @@
"""The host_agent plugin's MCP tools and its get_overview findings."""
from __future__ import annotations
from datetime import timedelta
from sqlalchemy import select
from steward.hosts import service as host_svc
from steward.mcp._context import mcp_app
from steward.mcp.tools._shape import age_s, db_session, iso, now_utc
from steward.mcp.tools.overview import finding
from steward.models.hosts import Host
from steward.models.users import UserRole
from .models import HostAgentRegistration
# A registration this old that has never checked in is a failed install, not one in progress.
PENDING_GRACE = timedelta(minutes=30)
def agent_state(reg, now, stale_after_s: int) -> str:
"""The agent's state: reporting, stale (stopped checking in) or pending (never has)."""
if reg.last_seen_at is None:
return "pending"
return "reporting" if (now - reg.last_seen_at).total_seconds() <= stale_after_s else "stale"
async def _agents(db):
return (await db.execute(
select(HostAgentRegistration, Host)
.join(Host, Host.id == HostAgentRegistration.host_id)
.order_by(Host.name)
)).all()
async def list_agents() -> dict:
"""Every host agent Steward has registered, with its version and whether it is reporting.
`state` is "reporting", "stale" (it checked in before but its last
check-in is older than the stale window, `stale_after_s`) or "pending"
(registered, never checked in, usually an install that is still running
or that failed). Each row carries the agent version, kernel, distro, arch
and the IP the agent reports. get_host shows an agent's latest metrics.
"""
now = now_utc()
stale_after = host_svc.agent_stale_after_seconds(mcp_app().config)
async with db_session() as db:
rows = await _agents(db)
agents = [
{
"host_id": h.id, "host_name": h.name,
"state": agent_state(reg, now, stale_after),
"last_seen_at": iso(reg.last_seen_at), "last_seen_age_s": age_s(reg.last_seen_at, now),
"agent_version": reg.agent_version, "kernel": reg.kernel, "distro": reg.distro,
"arch": reg.arch, "reported_ip": reg.host_ip,
"registered_at": iso(reg.created_at),
}
for reg, h in rows
]
counts = {s: sum(1 for a in agents if a["state"] == s) for s in ("reporting", "stale", "pending")}
return {"stale_after_s": stale_after, "counts": counts, "agents": agents, "total": len(agents)}
async def mcp_overview(db) -> list[dict]:
"""Agents registered more than PENDING_GRACE ago that have never checked in.
Agents that stopped reporting are already flagged by the core overview
from their metrics, so they are not repeated here.
"""
now = now_utc()
return [
finding("warning", "agent_never_reported", h.name,
"host agent was registered but has never checked in; the install may have failed",
since=reg.created_at, ids={"host_id": h.id}, next="list_agents()")
for reg, h in await _agents(db)
if reg.last_seen_at is None and now - reg.created_at > PENDING_GRACE
]
def register_tools(register) -> None:
register(list_agents, role=UserRole.viewer, read_only=True)
plugins/host_agent/metrics_query.py
+23 -7
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
from datetime import datetime, timedelta, timezone
from sqlalchemy import func, select
from sqlalchemy import func, or_, select
from steward.core.time_range import bucket_seconds
from steward.models.metrics import PluginMetric, PluginMetricHourly
@@ -27,14 +27,30 @@ HISTORY_METRICS = (
async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]:
"""{resource_name: {metric: value}} — latest sample per (resource, metric) for
a host + its sub-resources, from plugin_metric_latest (#5208). A host whose
agent has gone quiet keeps its last values until they age out of the raw
window; the caller decides staleness from the registration's last_seen_at.
"""
from steward.core.metric_series import latest_values
a host + its sub-resources.
DISTINCT ON picks the newest row per group in one index-ordered pass over
ix_plugin_metrics_module_resource_metric_recorded, instead of a GROUP-BY-max
subquery self-joined back to the table (two passes over the whole history).
"""
rows = (await session.execute(
select(PluginMetric)
.where(
PluginMetric.source_module == SOURCE_MODULE,
or_(
PluginMetric.resource_name == host_name,
PluginMetric.resource_name.like(host_name + ":%"),
),
)
.distinct(PluginMetric.resource_name, PluginMetric.metric_name)
.order_by(
PluginMetric.resource_name,
PluginMetric.metric_name,
PluginMetric.recorded_at.desc(),
)
)).scalars().all()
out: dict[str, dict[str, float]] = {}
for r in await latest_values(session, SOURCE_MODULE, host_name, include_children=True):
for r in rows:
out.setdefault(r.resource_name, {})[r.metric_name] = r.value
return out
plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py
-24
View File
@@ -1,24 +0,0 @@
# plugins/host_agent/migrations/versions/host_agent_003_clock_offset.py
"""host_agent: how far each host's clock is from Steward's (milestone 490)
Revision ID: host_agent_003_clock_offset
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "host_agent_003_clock_offset"
down_revision: Union[str, None] = "host_agent_002_host_ip"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("host_agent_registrations", sa.Column("clock_offset_s", sa.Float(), nullable=True))
op.add_column("host_agent_registrations",
sa.Column("clock_checked_at", sa.DateTime(timezone=True), nullable=True))
def downgrade() -> None:
op.drop_column("host_agent_registrations", "clock_checked_at")
op.drop_column("host_agent_registrations", "clock_offset_s")
plugins/host_agent/models.py
+1 -6
View File
@@ -2,7 +2,7 @@
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from sqlalchemy import Column, String, DateTime, Float, ForeignKey
from sqlalchemy import Column, String, DateTime, ForeignKey
from steward.models.base import Base
@@ -29,11 +29,6 @@ class HostAgentRegistration(Base):
# Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix).
host_ip = Column(String(45), nullable=True)
last_seen_at = Column(DateTime(timezone=True), nullable=True)
# How far this host's clock is from Steward's, in seconds, positive when
# the host is ahead, measured from the agent's sent_at on the newest post;
# and when that was. None until an agent that stamps its posts reports.
clock_offset_s = Column(Float, nullable=True)
clock_checked_at = Column(DateTime(timezone=True), nullable=True)
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
updated_at = Column(DateTime(timezone=True), nullable=False,
default=_utcnow, onupdate=_utcnow)
plugins/host_agent/routes.py
+263 -323
View File
@@ -1,9 +1,7 @@
# plugins/host_agent/routes.py
from __future__ import annotations
import asyncio
import hashlib
import time
from datetime import datetime, timezone
from typing import Any
@@ -11,21 +9,16 @@ from pathlib import Path
import secrets
from quart import Blueprint, current_app, jsonify, request, Response, render_template, redirect, url_for, session
from steward.auth.middleware import public_route, require_role
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from sqlalchemy import select, or_, and_
from sqlalchemy import select, func, or_, and_
from datetime import timedelta
from steward.core.formatting import worst_reading
from steward.core.health import ingest as ingest_tally
from steward.core.metric_series import points as metric_points
from steward.core.settings import get_setting, public_base_url
from steward.core.time_range import parse_range, RANGE_OPTIONS
from steward.hosts import service as host_svc
from steward.models.hosts import Host
from steward.models.metrics import PluginMetric
from .models import HostAgentRegistration
from .scheduler import arrivals
# Query helpers live in a model-free module so integration tests can import them
# without the plugin-loader double-registration guard tripping (see metrics_query).
from .metrics_query import (
@@ -47,35 +40,6 @@ def _parse_ts(ts: str) -> datetime:
return datetime.fromisoformat(ts)
def clock_offset(sent_at, received_at: datetime) -> float | None:
"""Seconds this host's clock is ahead of Steward's (negative: behind),
from the agent's sent_at and the moment Steward received the post. It
carries the request's own latency, well under a second on a LAN. None
when the post has no usable sent_at (an agent older than 1.12.0)."""
if not isinstance(sent_at, str):
return None
try:
sent = _parse_ts(sent_at)
except ValueError:
return None
if sent.tzinfo is None:
return None
return round((sent - received_at).total_seconds(), 2)
def clock_words(offset: float | None, thresholds: dict) -> tuple[str, str]:
"""('clock 0.3s ahead', 'ok') or ('clock 2m 10s behind', 'warn'): how far
a host's clock is from Steward's, graded by Settings → Thresholds' clock
drift, either way."""
from steward.core.formatting import human_duration, status_class
if offset is None:
return "", "unknown"
off = abs(offset)
words = "clock in step" if off < 1 else \
f"clock {human_duration(off)} {'ahead' if offset > 0 else 'behind'}"
return words, status_class(off, "clock", thresholds)
def pick_host_address(current: str | None, reported: str | None) -> str | None:
"""Return the address to store on the Host, or None for no change.
@@ -119,10 +83,6 @@ def _expand_sample_to_metrics(
for key in ("1m", "5m", "15m"):
if key in load:
row(f"load_{key}", host_name, load[key])
# Load per core as a %, the figure the pages grade, so it can learn a
# normal (#5742): a load of 6 is idle on 32 cores and busy on 4.
if "1m" in load and sample.get("cpu_cores"):
row("load_pct", host_name, 100.0 * load["1m"] / len(sample["cpu_cores"]))
if sample.get("uptime_secs") is not None:
row("uptime_secs", host_name, sample["uptime_secs"])
@@ -206,10 +166,7 @@ def _error(status: int, code: str, detail: str | None = None) -> tuple[Any, int]
@host_agent_bp.post("/ingest")
@public_route("the agent authenticates with its bearer token")
async def ingest():
# Taken first, so the clock offset carries as little of this handler as it can.
received_at = datetime.now(timezone.utc)
auth = request.headers.get("Authorization", "")
if not auth.startswith("Bearer "):
return _error(401, "invalid_token")
@@ -229,226 +186,110 @@ async def ingest():
return _error(400, "malformed_payload", "'samples' must be a non-empty list")
token_hash = _hash_token(raw)
phases = _Phases()
samples, dropped = newest_samples(samples, MAX_SAMPLES_PER_INGEST)
if dropped:
current_app.logger.warning(
"host_agent ingest: kept the newest %d of %d samples for token=%s…",
len(samples), len(samples) + dropped, token_hash[:8])
try:
async with current_app.db_sessionmaker() as session:
reg = (await session.execute(
select(HostAgentRegistration).where(
HostAgentRegistration.token_hash == token_hash)
)).scalar_one_or_none()
if reg is None:
return _error(401, "invalid_token")
host = (await session.execute(
select(Host).where(Host.id == reg.host_id)
)).scalar_one_or_none()
if host is None:
return _error(401, "invalid_token")
# The agent reached us: that holds even if storing its post fails (#5514).
arrivals(current_app)[reg.host_id] = received_at
async with current_app.db_sessionmaker() as session:
reg = (await session.execute(
select(HostAgentRegistration).where(
HostAgentRegistration.token_hash == token_hash)
)).scalar_one_or_none()
if reg is None:
return _error(401, "invalid_token")
host = (await session.execute(
select(Host).where(Host.id == reg.host_id)
)).scalar_one_or_none()
if host is None:
return _error(401, "invalid_token")
accepted = 0
metric_rows: list[PluginMetric] = []
latest_ts: datetime | None = None
docker_snapshots: list[tuple[datetime, list]] = []
docker_log_batches: list[tuple[datetime, list]] = []
latest_swarm: dict | None = None
latest_swarm_ts: datetime | None = None
latest_disk: dict | None = None
latest_disk_ts: datetime | None = None
# Whether Docker answered, from the newest sample that says (agent ≥ 1.9.0).
docker_status: dict | None = None
for sample in samples:
accepted = 0
latest_ts: datetime | None = None
docker_snapshots: list[tuple[datetime, list]] = []
latest_swarm: dict | None = None
latest_swarm_ts: datetime | None = None
latest_disk: dict | None = None
latest_disk_ts: datetime | None = None
for sample in samples:
try:
recorded_at = _parse_ts(sample["ts"])
except (KeyError, ValueError):
continue
metrics = _expand_sample_to_metrics(sample, host.name, recorded_at)
for m in metrics:
session.add(m)
docker = sample.get("docker")
if isinstance(docker, list) and docker:
docker_snapshots.append((recorded_at, docker))
# Swarm is current-state, not time-series — keep only the newest
# sample's topology (a manager re-reports it every interval).
swarm = sample.get("swarm")
if isinstance(swarm, dict) and (
latest_swarm_ts is None or recorded_at > latest_swarm_ts):
latest_swarm, latest_swarm_ts = swarm, recorded_at
# Disk usage is current-state too — keep only the newest sample's.
disk = sample.get("docker_disk")
if isinstance(disk, dict) and (
latest_disk_ts is None or recorded_at > latest_disk_ts):
latest_disk, latest_disk_ts = disk, recorded_at
accepted += 1
if latest_ts is None or recorded_at > latest_ts:
latest_ts = recorded_at
if accepted == 0:
await session.rollback()
return _error(400, "malformed_payload", "no valid samples")
# Hand host-scoped container data to the docker plugin if it's enabled
# (opportunistic synergy via the capability registry — no hard import,
# no-op when docker is disabled). A failure here must never sink the
# whole ingest, so the metrics above still land.
if docker_snapshots or latest_swarm is not None or latest_disk is not None:
from steward.core.capabilities import has_capability, invoke_capability
if has_capability("docker.persist_host_samples"):
try:
recorded_at = _parse_ts(sample["ts"])
except (KeyError, ValueError):
continue
metric_rows.extend(_expand_sample_to_metrics(sample, host.name, recorded_at))
# An empty list is a real answer: Docker is up with no containers.
docker = sample.get("docker")
if isinstance(docker, list):
docker_snapshots.append((recorded_at, docker))
status = sample.get("docker_status")
if status in ("ok", "unavailable", "error") and (
docker_status is None or recorded_at > docker_status["at"]):
docker_status = {
"status": status, "detail": sample.get("docker_detail"),
"at": recorded_at, "swarm_manager": sample.get("swarm_manager")}
# Container logs are time-series (append every line, not newest-only);
# each record carries its own Docker ts, recorded_at is the fallback.
docker_logs = sample.get("docker_logs")
if isinstance(docker_logs, list) and docker_logs:
docker_log_batches.append((recorded_at, docker_logs))
# Swarm is current-state, not time-series — keep only the newest
# sample's topology (a manager re-reports it every interval).
swarm = sample.get("swarm")
if isinstance(swarm, dict) and (
latest_swarm_ts is None or recorded_at > latest_swarm_ts):
latest_swarm, latest_swarm_ts = swarm, recorded_at
# Disk usage is current-state too — keep only the newest sample's.
disk = sample.get("docker_disk")
if isinstance(disk, dict) and (
latest_disk_ts is None or recorded_at > latest_disk_ts):
latest_disk, latest_disk_ts = disk, recorded_at
accepted += 1
if latest_ts is None or recorded_at > latest_ts:
latest_ts = recorded_at
# SAVEPOINT so a docker-side failure rolls back only the
# docker writes, leaving the host metrics intact to commit.
async with session.begin_nested():
await invoke_capability(
"docker.persist_host_samples", UserRole.admin,
session, host, docker_snapshots, latest_swarm,
latest_disk,
)
except Exception:
current_app.logger.exception(
"host_agent ingest: docker persist failed for host=%s",
host.name)
if accepted == 0:
await session.rollback()
return _error(400, "malformed_payload", "no valid samples")
# How far this host's clock is from Steward's, either way, so an
# alert rule reads "> 30" (milestone 490).
offset = clock_offset(payload.get("sent_at"), received_at)
if offset is not None:
metric_rows.append(PluginMetric(
source_module=SOURCE_MODULE, resource_name=host.name,
metric_name="clock_off_secs", value=abs(offset), recorded_at=received_at))
phases.mark("parse")
# Through the alert pipeline, so rules on host metrics can fire.
from steward.core.alerts import record_metrics
phases.size = f"samples={accepted} metric_rows={len(metric_rows)}"
await record_metrics(session, metric_rows, phases)
# Hand host-scoped container data to the docker plugin if it's enabled
# (opportunistic synergy via the capability registry — no hard import,
# no-op when docker is disabled). A failure here must never sink the
# whole ingest, so the metrics above still land.
if (docker_snapshots or latest_swarm is not None or docker_status is not None
or latest_disk is not None or docker_log_batches):
from steward.core.capabilities import has_capability, invoke_capability
if has_capability("docker.persist_host_samples"):
try:
# SAVEPOINT so a docker-side failure rolls back only the
# docker writes, leaving the host metrics intact to commit.
async with session.begin_nested():
await invoke_capability(
"docker.persist_host_samples", UserRole.admin,
session, host, docker_snapshots, latest_swarm,
latest_disk, docker_log_batches, docker_status,
phases=phases,
)
except Exception:
current_app.logger.exception(
"host_agent ingest: docker persist failed for host=%s",
host.name)
phases.mark("docker")
md = payload.get("metadata") or {}
changed = False
if latest_ts and (reg.last_seen_at is None or latest_ts > reg.last_seen_at):
reg.last_seen_at = latest_ts
md = payload.get("metadata") or {}
changed = False
if latest_ts and (reg.last_seen_at is None or latest_ts > reg.last_seen_at):
reg.last_seen_at = latest_ts
changed = True
version = payload.get("agent_version")
if version and reg.agent_version != version:
reg.agent_version = version
changed = True
for field in ("kernel", "distro", "arch"):
if field in md and getattr(reg, field) != md[field]:
setattr(reg, field, md[field])
changed = True
version = payload.get("agent_version")
if version and reg.agent_version != version:
reg.agent_version = version
changed = True
for field in ("kernel", "distro", "arch"):
if field in md and getattr(reg, field) != md[field]:
setattr(reg, field, md[field])
changed = True
host_ip = md.get("host_ip")
if host_ip and reg.host_ip != host_ip:
reg.host_ip = host_ip
changed = True
# Mirror the reported IP into Host.address only when it's blank.
new_address = pick_host_address(host.address, host_ip)
if new_address is not None:
host.address = new_address
if changed:
reg.updated_at = datetime.now(timezone.utc)
host_ip = md.get("host_ip")
if host_ip and reg.host_ip != host_ip:
reg.host_ip = host_ip
changed = True
# Mirror the reported IP into Host.address only when it's blank.
new_address = pick_host_address(host.address, host_ip)
if new_address is not None:
host.address = new_address
if changed:
reg.updated_at = datetime.now(timezone.utc)
if offset is not None:
reg.clock_offset_s, reg.clock_checked_at = offset, received_at
clock_th = current_app.config.get("THRESHOLDS", {}).get("clock") or {"warn": 30}
if abs(offset) > clock_th["warn"]:
current_app.logger.warning(
"host_agent ingest: host=%s clock is %.1fs %s Steward's",
host.name, abs(offset), "ahead of" if offset > 0 else "behind")
if latest_ts:
skew = abs((datetime.now(timezone.utc) - latest_ts).total_seconds())
if skew > 300:
current_app.logger.warning(
"host_agent ingest: clock skew %.0fs for host=%s", skew, host.name)
await session.commit()
phases.mark("commit")
except asyncio.CancelledError:
# The agent hung up (its 10 s timeout) and the server dropped the
# request: say how far it got before re-raising.
phases.cancelled(token_hash[:8])
ingest_tally.done(phases.elapsed_ms(), ok=False)
raise
except Exception:
ingest_tally.done(phases.elapsed_ms(), ok=False)
raise
# Counted for Steward's own health (#5467). The early returns above (a
# bad token, no valid samples) answer the agent and aren't counted.
ingest_tally.done(phases.elapsed_ms())
await session.commit()
phases.report(host.name, accepted, len(metric_rows),
sum(len(c) for _, c in docker_snapshots),
sum(len(lines) for _, lines in docker_log_batches))
return jsonify({"ok": True, "samples_accepted": accepted, "samples_dropped": dropped}), 200
# The agent gives up on a post after 10 s (agent.py); an ingest past this
# says where its time went, so a slow phase shows in the logs (#5149).
SLOW_INGEST_SECONDS = 2.0
# How many samples one ingest writes (#5149). An agent that could not post
# sends its whole buffer (up to 21 samples, ~4,000 metric rows); on the live
# instance that did not finish inside the agent's 10 s, the rollback left the
# buffer full, and every later post failed the same way. The newest samples
# are kept: the older ones would be lost to the agent's ring buffer anyway.
# Five matches the agent's MAX_SAMPLES_PER_POST (1.10+), so its backlog posts
# land whole: ~1,000 metric rows at the 1,300-4,000 rows/s measured once
# plugin_metrics got time-ordered ids, with posts answering in 0.4-1.3 s (#5181).
MAX_SAMPLES_PER_INGEST = 5
def newest_samples(samples: list, limit: int) -> tuple[list, int]:
"""(the newest `limit` samples, oldest first; how many were left out).
Ordered by their ISO-8601 `ts` (the agent writes them all in one form);
one without a ts sorts oldest and is rejected later as malformed."""
if len(samples) <= limit:
return samples, 0
ordered = sorted(samples, key=lambda s: str(s.get("ts") or "") if isinstance(s, dict) else "")
return ordered[-limit:], len(samples) - limit
class _Phases:
"""Wall-clock per phase of one ingest request."""
def __init__(self):
self.start = self.last = time.monotonic()
self.took: dict[str, float] = {}
self.size = ""
def elapsed_ms(self) -> float:
return (time.monotonic() - self.start) * 1000.0
def mark(self, name: str) -> None:
now = time.monotonic()
self.took[name] = self.took.get(name, 0.0) + (now - self.last)
self.last = now
def cancelled(self, who: str) -> None:
current_app.logger.warning(
"host_agent ingest: dropped after %.1fs for token=%s… (completed: %s) %s",
time.monotonic() - self.start, who,
" ".join(f"{k}={v:.1f}s" for k, v in self.took.items()) or "nothing", self.size)
def report(self, host: str, samples: int, metrics: int, containers: int, log_lines: int) -> None:
total = time.monotonic() - self.start
if total < SLOW_INGEST_SECONDS:
return
current_app.logger.warning(
"host_agent ingest: slow %.1fs for host=%s (%s) samples=%d metric_rows=%d "
"containers=%d log_lines=%d", total, host,
" ".join(f"{k}={v:.1f}s" for k, v in self.took.items()),
samples, metrics, containers, log_lines)
return jsonify({"ok": True, "samples_accepted": accepted}), 200
AGENT_SOURCE_PATH = Path(__file__).parent / "agent.py"
@@ -465,7 +306,6 @@ def _agent_version() -> str:
@host_agent_bp.get("/install.sh")
@public_route("fetched by curl on the host being enrolled; carries an enrolment token")
async def install_script():
token = request.args.get("token", "")
if not token:
@@ -497,7 +337,6 @@ async def install_script():
@host_agent_bp.get("/agent.py")
@public_route("fetched by the installer and by agent self-update; holds no secrets")
async def agent_source():
try:
body = AGENT_SOURCE_PATH.read_text(encoding="utf-8")
@@ -506,28 +345,6 @@ async def agent_source():
return Response(body, mimetype="text/x-python")
_FLEET_ORDER = {"stale": 0, "pending": 1, "up": 2}
def fleet_state(reg, now: datetime, stale_after: int) -> str:
"""An agent as the fleet table shows it: "up" while it reports, "stale"
once its last report is older than stale_after, "pending" if it never has."""
if reg.last_seen_at is None:
return "pending"
return "stale" if (now - reg.last_seen_at).total_seconds() > stale_after else "up"
def fleet_rows(regs, hosts_by_id: dict, now: datetime, stale_after: int) -> list[dict]:
"""The fleet table: silent agents first, then never-reported, then live,
each by host name."""
rows = []
for r in regs:
host = hosts_by_id.get(r.host_id)
rows.append({"reg": r, "host": host, "name": host.name if host else r.host_id,
"state": fleet_state(r, now, stale_after)})
return sorted(rows, key=lambda x: (_FLEET_ORDER[x["state"]], x["name"].lower()))
def _stale_after_seconds() -> int:
return int(
current_app.config.get("PLUGINS", {})
@@ -536,8 +353,124 @@ def _stale_after_seconds() -> int:
)
async def _fleet_rows(session) -> list[dict]:
"""One row per registered host with its latest host-level metrics + stale flag.
Only bare host-level resources are used here (sub-resources carry a ':' and
are skipped) so the fleet glance stays one line per host.
"""
regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
host_ids = [r.host_id for r in regs]
hosts = {
h.id: h for h in (await session.execute(
select(Host).where(Host.id.in_(host_ids))
)).scalars().all()
} if host_ids else {}
subq = (
select(
PluginMetric.resource_name,
PluginMetric.metric_name,
func.max(PluginMetric.recorded_at).label("max_ts"),
)
.where(PluginMetric.source_module == SOURCE_MODULE)
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
).subquery()
latest_rows = (await session.execute(
select(PluginMetric).join(
subq,
(PluginMetric.resource_name == subq.c.resource_name) &
(PluginMetric.metric_name == subq.c.metric_name) &
(PluginMetric.recorded_at == subq.c.max_ts),
).where(PluginMetric.source_module == SOURCE_MODULE)
)).scalars().all()
latest: dict[str, dict[str, float]] = {}
root_disk: dict[str, float] = {} # host_name → root (/) disk %
for row in latest_rows:
if row.resource_name.endswith(":/") and row.metric_name == "disk_used_pct":
root_disk[row.resource_name[:-2]] = row.value
if ":" in row.resource_name:
continue
latest.setdefault(row.resource_name, {})[row.metric_name] = row.value
# Per-host recent series for the inline sparklines (cpu/mem/load host-level,
# disk from the root mount), so each metric shows its trend beside the value —
# the host-panel at-a-glance, on the fleet widget. 1h window keeps it cheap.
host_names = [h.name for h in hosts.values()]
spark_series: dict[str, dict[str, list]] = {
n: {"cpu": [], "mem": [], "disk": [], "load": []} for n in host_names
}
if host_names:
since = datetime.now(timezone.utc) - timedelta(hours=1)
sp_rows = (await session.execute(
select(PluginMetric).where(
PluginMetric.source_module == SOURCE_MODULE,
PluginMetric.recorded_at >= since,
or_(
and_(PluginMetric.resource_name.in_(host_names),
PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))),
and_(PluginMetric.resource_name.in_([n + ":/" for n in host_names]),
PluginMetric.metric_name == "disk_used_pct"),
),
).order_by(PluginMetric.recorded_at)
)).scalars().all()
for r in sp_rows:
if r.resource_name.endswith(":/"):
name = r.resource_name[:-2]
if name in spark_series:
spark_series[name]["disk"].append(r.value)
else:
key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name)
if key and r.resource_name in spark_series:
spark_series[r.resource_name][key].append(r.value)
from steward.core.status import sparkline_svg
stale_after = _stale_after_seconds()
now = datetime.now(timezone.utc)
rows = []
for reg in regs:
host = hosts.get(reg.host_id)
if host is None:
continue
m = latest.get(host.name, {})
ls = reg.last_seen_at
stale = ls is None or (now - ls).total_seconds() > stale_after
ser = spark_series.get(host.name, {})
sparks = {
k: (sparkline_svg(v[-60:], width=72, height=16) if len(v) >= 2 else "")
for k, v in ser.items()
}
rows.append({
"host": host,
"reg": reg,
"stale": stale,
"cpu_pct": m.get("cpu_pct"),
"mem_used_pct": m.get("mem_used_pct"),
"disk_worst": m.get("disk_used_pct_worst"), # health dot only
"disk_root": root_disk.get(host.name), # displayed at-a-glance
"load_1m": m.get("load_1m"),
"temp_max": m.get("temp_c_max"),
"net_rx_bps": m.get("net_rx_bps"),
"net_tx_bps": m.get("net_tx_bps"),
"disk_read_bps": m.get("disk_read_bps"),
"disk_write_bps": m.get("disk_write_bps"),
"sparks": sparks,
})
rows.sort(key=lambda r: r["host"].name.lower())
return rows
@host_agent_bp.get("/widget")
async def widget_table():
"""Fleet-glance dashboard widget: one row per monitored host, latest metrics."""
async with current_app.db_sessionmaker() as session:
rows = await _fleet_rows(session)
return await render_template("widget_table.html", rows=rows)
@host_agent_bp.get("/widget/history")
@require_role(UserRole.viewer)
async def widget_history():
host_id = request.args.get("host_id", "")
try:
@@ -730,27 +663,38 @@ async def host_vitals(host_id: str):
reg = (await db.execute(select(HostAgentRegistration).where(
HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
latest = await _latest_metrics_for_host(db, host.name) if reg else {}
norms = await host_svc.host_norms(db, host.name) if reg else {}
# The last 6 h for the at-a-glance sparklines (cpu/mem/load host-level,
# disk from the root mount), averaged into ~120 points in SQL, so the
# line spans the whole window and only what is drawn is fetched.
# Recent series for the at-a-glance sparklines (cpu/mem/load host-level,
# disk from the root mount sub-resource).
series_raw: dict[str, list[float]] = {"cpu": [], "mem": [], "disk": [], "load": []}
if reg:
since = datetime.now(timezone.utc) - timedelta(hours=6)
for key, resource, metric in (("cpu", host.name, "cpu_pct"),
("mem", host.name, "mem_used_pct"),
("load", host.name, "load_1m"),
("disk", host.name + ":/", "disk_used_pct")):
series_raw[key] = [v for _t, v in await metric_points(
db, SOURCE_MODULE, resource, metric, since, target=120)]
spark_rows = (await db.execute(
select(PluginMetric).where(
PluginMetric.source_module == SOURCE_MODULE,
PluginMetric.recorded_at >= since,
or_(
and_(PluginMetric.resource_name == host.name,
PluginMetric.metric_name.in_(("cpu_pct", "mem_used_pct", "load_1m"))),
and_(PluginMetric.resource_name == host.name + ":/",
PluginMetric.metric_name == "disk_used_pct"),
),
).order_by(PluginMetric.recorded_at)
)).scalars().all()
for r in spark_rows:
if r.resource_name == host.name:
key = {"cpu_pct": "cpu", "mem_used_pct": "mem", "load_1m": "load"}.get(r.metric_name)
if key:
series_raw[key].append(r.value)
else:
series_raw["disk"].append(r.value)
from steward.core.status import sparkline_svg
hostlvl = latest.get(host.name, {})
# At-a-glance disk = the root filesystem (what people actually care about),
# not the "worst" mount which is opaque. Worst stays on the full-metrics page.
disk_root = (latest.get(host.name + ":/", {}) or {}).get("disk_used_pct")
sparks = {k: sparkline_svg(v, height=24, fill_width=True) for k, v in series_raw.items()}
sparks = {k: sparkline_svg(v[-120:]) for k, v in series_raw.items()}
# Normalize load by core count so it's comparable across hardware (load/core
# as %). Cores derived from the per-core CPU sub-resources already collected.
cores = sum(1 for k in latest if k.startswith(host.name + ":core"))
@@ -764,25 +708,11 @@ async def host_vitals(host_id: str):
ls = reg.last_seen_at if reg else None
reporting = bool(reg) and ls is not None
stale = reporting and (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
cpu, mem = hostlvl.get("cpu_pct"), hostlvl.get("mem_used_pct")
thresholds = current_app.config.get("THRESHOLDS", {})
worst = worst_reading((("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("Disk /", disk_root, "disk"),
("Load per core", load_per_core, "load")),
thresholds, host_svc.host_normals(norms, thresholds))
# Each tile's grade; CPU, memory and load within the host's own normal
# for the hour read healthy, and say what is usual (#5742).
grades = host_svc.host_grades({"cpu": cpu, "mem": mem, "disk": disk_root, "load": load_per_core},
norms, thresholds)
usual = [f"{label} {value:.0f}% is {grades[kind][1]}"
for label, value, kind in (("CPU", cpu, "cpu"), ("Memory", mem, "mem"),
("load per core", load_per_core, "load"))
if grades[kind][1]]
return await render_template(
"_host_vitals.html",
reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core,
sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst,
grades=grades, usual=usual,
clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})),
reg=reg, cpu=hostlvl.get("cpu_pct"), mem=hostlvl.get("mem_used_pct"),
disk_root=disk_root, load_per_core=load_per_core, sparks=sparks, psi=psi,
reporting=reporting, stale=stale,
)
@@ -854,10 +784,11 @@ async def fleet():
managed_key_set = bool(
(current_app.config.get("ANSIBLE", {}).get("ssh_public_key") or "").strip())
now, stale_after = datetime.now(timezone.utc), _stale_after_seconds()
return await render_template(
"settings_list.html",
registrations=fleet_rows(regs, hosts_by_id, now, stale_after),
registrations=[
{"reg": r, "host": hosts_by_id.get(r.host_id)} for r in regs
],
new_token=new_token,
install_url=install_url,
ansible_available=ansible_available,
@@ -867,24 +798,35 @@ async def fleet():
)
@host_agent_bp.post("/fleet/<host_id>/register")
@host_agent_bp.post("/fleet/add-host")
@require_role(UserRole.admin)
async def register_host(host_id: str):
"""Issue an agent token for an existing host, for a manual install: the
fleet page then shows the one-line install command, once. Hosts are added
on Hosts → Add host; this is only the agent half (#5282)."""
async def add_host():
form = await request.form
name = (form.get("name") or "").strip()
address = (form.get("address") or "").strip()
if not name:
return _error(400, "missing_name")
async with current_app.db_sessionmaker() as session:
host = await session.get(Host, host_id)
host = (await session.execute(
select(Host).where(Host.name == name))).scalar_one_or_none()
if host is None:
return _error(404, "not_found")
host = Host(name=name, address=address)
session.add(host)
await session.flush()
existing = (await session.execute(
select(HostAgentRegistration).where(
HostAgentRegistration.host_id == host.id))).scalar_one_or_none()
if existing is not None:
await session.rollback()
return _error(400, "already_registered")
raw, hashed = _new_token_pair()
session.add(HostAgentRegistration(host_id=host.id, token_hash=hashed))
reg = HostAgentRegistration(host_id=host.id, token_hash=hashed)
session.add(reg)
await session.commit()
host_id = host.id
return redirect(url_for("host_agent.fleet") + f"?new_token={raw}&host_id={host_id}")
@@ -1005,7 +947,6 @@ async def deploy_via_ansible():
playbook_path="host_agent/install.yml",
inventory_content=inventory_content,
inventory_scope=scope,
redact=list(tokens.values()),
# extra-vars outrank the playbook's `vars:` defaults; token stays per-host.
params={"extra_vars_map": {"steward_url": url, "agent_interval": interval}},
triggered_by=session.get("user_id"),
@@ -1083,7 +1024,6 @@ async def provision_via_ansible():
playbook_path="host_agent/provision.yml",
inventory_content=inventory_content,
inventory_scope=scope,
redact=list(tokens.values()),
# extra_vars_map → a JSON -e @file: highest precedence (beats play vars)
# and space-safe (steward_pubkey carries a comment with a space).
params={"extra_vars_map": {
plugins/host_agent/scheduler.py
+51 -84
View File
@@ -1,111 +1,78 @@
# plugins/host_agent/scheduler.py
from __future__ import annotations
import logging
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from typing import Iterable
from sqlalchemy import select
from steward.core.alerts import MetricBatch
from steward.core.scheduler import ScheduledTask
from steward.core.watch import watch
from steward.hosts.service import agent_stale_after_seconds
from steward.models.hosts import Host
from steward.models.metrics import PluginMetricLatest
from .models import HostAgentRegistration
logger = logging.getLogger(__name__)
def arrivals(app) -> dict[str, datetime]:
"""{host id: when its agent's newest post reached Steward}, in memory (#5514).
last_seen_at is written in the ingest's own transaction, so when Steward
can't store a post (its storage stalls and the request is dropped) the
agent's report is lost with it, and an agent posting every few seconds read
as offline. A post that arrived is the agent reaching Steward; storing it
is Steward's part. Set before any write, so it holds however slow they are.
"""
got = getattr(app, "_agent_arrivals", None)
if got is None:
got = app._agent_arrivals = {}
return got
def _filter_stale(
regs: Iterable,
*,
now: datetime,
stale_after_seconds: int,
) -> list:
"""Pure staleness filter: returns the subset with last_seen_at strictly
older than (now - stale_after_seconds). Rows with last_seen_at=None are
never stale (they are unregistered-in-practice)."""
cutoff = now - timedelta(seconds=stale_after_seconds)
return [
r for r in regs
if r.last_seen_at is not None and r.last_seen_at < cutoff
]
def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
now: datetime, watching_since: datetime | None = None,
arrived: dict[str, datetime] | None = None,
already_silent: set[str] | frozenset = frozenset(),
) -> list[tuple[str, float]]:
"""(host name, seconds since its agent last reported) per registration.
An agent that has never reported counts from when it was registered. A
registration whose host is gone is skipped. Nothing counts from before
`watching_since`, when Steward last started watching without a break: an
agent can't report while Steward is down, so that time is not its silence
(#5441). An agent that is really gone reads as offline once it has been
quiet that long while Steward watched. `arrived` (arrivals(app)) counts a
post Steward received but couldn't store as the agent seen (#5514).
`already_silent` names hosts whose last recorded figure was already past
the offline window: their agent was gone before Steward stopped, so their
whole silence counts. Counting them from `watching_since` read them as
just seen, which resolved their alert on every restart and fired it again
three minutes later (#5719).
"""
out = []
for r in regs:
name = host_names.get(r.host_id)
seen = max((t for t in (r.last_seen_at or r.created_at, (arrived or {}).get(r.host_id))
if t is not None), default=None)
if name is None or seen is None:
continue
if watching_since is not None and name not in already_silent:
seen = max(seen, watching_since)
out.append((name, max(0.0, (now - seen).total_seconds())))
return out
async def record_seconds_since_seen(app) -> None:
"""Record host_agent/<host>/seconds_since_seen for every registered agent,
so "agent offline" is an ordinary alert rule (seconds_since_seen > 180)."""
now = datetime.now(timezone.utc)
async def find_stale_registrations(app, stale_after_seconds: int = 180) -> list[dict]:
async with app.db_sessionmaker() as session:
async with session.begin():
regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
watching_since = watch(app, "host_agent_seconds_since_seen").tick(now)
if not regs:
return
host_names = dict((await session.execute(
select(Host.id, Host.name).where(Host.id.in_([r.host_id for r in regs]))
)).all())
# Agents whose last figure was already past the offline window keep
# counting through a restart of Steward (#5719).
stale_after = agent_stale_after_seconds(app.config)
already_silent = set((await session.execute(
select(PluginMetricLatest.resource_name).where(
PluginMetricLatest.source_module == "host_agent",
PluginMetricLatest.metric_name == "seconds_since_seen",
PluginMetricLatest.resource_name.in_(list(host_names.values())),
PluginMetricLatest.value > stale_after))).scalars())
batch = MetricBatch()
for name, seconds in seconds_since_seen(regs, host_names, now, watching_since,
arrivals(app), already_silent):
batch.add(source_module="host_agent", resource_name=name,
metric_name="seconds_since_seen", value=seconds, recorded_at=now)
await batch.record(session)
all_regs = (await session.execute(
select(HostAgentRegistration)
)).scalars().all()
stale_rows = _filter_stale(
all_regs,
now=datetime.now(timezone.utc),
stale_after_seconds=stale_after_seconds,
)
if not stale_rows:
return []
hosts = {
h.id: h for h in (await session.execute(
select(Host).where(Host.id.in_([r.host_id for r in stale_rows]))
)).scalars().all()
}
return [
{
"host_id": r.host_id,
"host_name": hosts[r.host_id].name if r.host_id in hosts else "?",
"last_seen_at": r.last_seen_at,
}
for r in stale_rows
]
def make_task(app) -> ScheduledTask:
async def _record():
async def _check_stale():
try:
await record_seconds_since_seen(app)
stale = await find_stale_registrations(app)
except Exception:
logger.exception("host_agent: recording seconds_since_seen failed")
logger.exception("host_agent stale check failed")
return
if stale:
logger.info(
"host_agent: %d stale agent(s): %s",
len(stale),
[s["host_name"] for s in stale],
)
return ScheduledTask(
name="host_agent_seconds_since_seen",
coro_factory=_record,
name="host_agent_stale_check",
coro_factory=_check_stale,
interval_seconds=60,
run_on_startup=False,
)
plugins/host_agent/templates/_host_metrics.html
+42 -25
View File
@@ -1,15 +1,32 @@
{# Current-state fragment for the full-metrics page — polled live via HTMX.
Each poll re-renders the latest snapshot the server has. #}
{% from "_macros.html" import status_badge %}
{# A percentage bar graded by Settings → Thresholds for `kind` (cpu, mem, disk). #}
{% macro bar(pct, kind) %}
<div class="meter"><div class="meter-fill meter-{{ status_class(pct, kind) }}" style="width:{{ (pct or 0)|round(1) }}%;"></div></div>
Self-contained (defines its own format macros) so each poll re-renders the
latest snapshot the server has. #}
{% macro fmt_bps(v) %}
{%- if v is none -%}—
{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s
{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s
{%- else -%}{{ "%.0f"|format(v) }} B/s
{%- endif -%}
{% endmacro %}
{% macro fmt_bytes(v) %}
{%- if v is none -%}—
{%- elif v >= 1125899906842624 -%}{{ "%.1f"|format(v / 1125899906842624) }} PB
{%- elif v >= 1099511627776 -%}{{ "%.1f"|format(v / 1099511627776) }} TB
{%- elif v >= 1073741824 -%}{{ "%.1f"|format(v / 1073741824) }} GB
{%- elif v >= 1048576 -%}{{ "%.0f"|format(v / 1048576) }} MB
{%- else -%}{{ "%.0f"|format(v / 1024) }} KB
{%- endif -%}
{% endmacro %}
{% macro bar(pct) %}
{%- set p = pct if pct is not none else 0 -%}
<div style="height:8px;background:var(--bg-elevated);border-radius:4px;overflow:hidden;border:1px solid var(--border);">
<div style="height:100%;width:{{ p|round(1) }}%;background:{% if pct is none %}var(--text-dim){% elif pct < 70 %}var(--green){% elif pct < 90 %}var(--yellow){% else %}var(--red){% endif %};"></div>
</div>
{% endmacro %}
{% macro gauge(value, kind, text) %}<span class="stat-val tone-{{ status_class(value, kind) }}" style="font-size:1.6rem;">{{ text }}</span>{% endmacro %}
{# ── Identity / metadata ─────────────────────────────────────────────────── #}
<div class="card" style="display:flex;flex-wrap:wrap;gap:0.5rem 1.25rem;align-items:center;font-size:0.82rem;color:var(--text-muted);">
<span>{{ status_badge('stale' if stale else 'up', 'Stale' if stale else 'Live') }}</span>
<span>{% if stale %}<span class="badge badge-red">stale</span>{% else %}<span class="badge badge-green">live</span>{% endif %}</span>
<span><span style="color:var(--text-dim);">Address</span> {{ host.address or "—" }}</span>
{% if reg %}
{% if reg.distro %}<span><span style="color:var(--text-dim);">OS</span> {{ reg.distro }}</span>{% endif %}
@@ -17,8 +34,8 @@
{% if reg.arch %}<span><span style="color:var(--text-dim);">Arch</span> {{ reg.arch }}</span>{% endif %}
{% if reg.agent_version %}<span><span style="color:var(--text-dim);">Agent</span> v{{ reg.agent_version }}</span>{% endif %}
{% set up = hostlvl.get('uptime_secs') %}
{% if up %}<span><span style="color:var(--text-dim);">Uptime</span> {{ up|duration }}</span>{% endif %}
<span><span style="color:var(--text-dim);">Last seen</span> {{ reg.last_seen_at|ago if reg.last_seen_at else "never" }}</span>
{% if up %}<span><span style="color:var(--text-dim);">Uptime</span> {{ (up // 86400)|int }}d {{ ((up % 86400) // 3600)|int }}h</span>{% endif %}
<span><span style="color:var(--text-dim);">Last seen</span> {{ reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") ~ " UTC" if reg.last_seen_at else "never" }}</span>
{% else %}
<span>No agent registration found for this host.</span>
{% endif %}
@@ -33,16 +50,16 @@
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">CPU</div>
{% set c = hostlvl.get('cpu_pct') %}
{{ gauge(c, 'cpu', ("%.0f%%"|format(c)) if c is not none else "—") }}
<div style="margin-top:0.5rem;">{{ bar(c, 'cpu') }}</div>
<span class="stat-val" style="font-size:1.6rem;color:{% if c is none %}var(--text-dim){% elif c < 70 %}var(--green){% elif c < 90 %}var(--yellow){% else %}var(--red){% endif %};">{% if c is not none %}{{ "%.0f"|format(c) }}%{% else %}—{% endif %}</span>
<div style="margin-top:0.5rem;">{{ bar(c) }}</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Memory</div>
{% set mp = hostlvl.get('mem_used_pct') %}
{{ gauge(mp, 'mem', ("%.0f%%"|format(mp)) if mp is not none else "—") }}
<div style="margin-top:0.5rem;">{{ bar(mp, 'mem') }}</div>
<span class="stat-val" style="font-size:1.6rem;color:{% if mp is none %}var(--text-dim){% elif mp < 70 %}var(--green){% elif mp < 90 %}var(--yellow){% else %}var(--red){% endif %};">{% if mp is not none %}{{ "%.0f"|format(mp) }}%{% else %}—{% endif %}</span>
<div style="margin-top:0.5rem;">{{ bar(mp) }}</div>
<div style="margin-top:0.45rem;font-size:0.72rem;color:var(--text-dim);">
avail {{ hostlvl.get('mem_available_bytes')|bytes }} · cache {{ hostlvl.get('mem_cached_bytes')|bytes }} · swap {{ hostlvl.get('swap_used_bytes')|bytes }}
avail {{ fmt_bytes(hostlvl.get('mem_available_bytes')) }} · cache {{ fmt_bytes(hostlvl.get('mem_cached_bytes')) }} · swap {{ fmt_bytes(hostlvl.get('swap_used_bytes')) }}
</div>
</div>
<div class="card" style="margin-bottom:0;">
@@ -56,22 +73,22 @@
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Network</div>
<div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
<div><span style="color:var(--green);">↓</span> {{ hostlvl.get('net_rx_bps')|rate }}</div>
<div><span style="color:var(--red);">↑</span> {{ hostlvl.get('net_tx_bps')|rate }}</div>
<div><span style="color:var(--green);">↓</span> {{ fmt_bps(hostlvl.get('net_rx_bps')) }}</div>
<div><span style="color:var(--red);">↑</span> {{ fmt_bps(hostlvl.get('net_tx_bps')) }}</div>
</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Disk I/O</div>
<div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
<div><span style="color:var(--text-dim);">rd</span> {{ hostlvl.get('disk_read_bps')|rate }}</div>
<div><span style="color:var(--text-dim);">wr</span> {{ hostlvl.get('disk_write_bps')|rate }}</div>
<div><span style="color:var(--text-dim);">rd</span> {{ fmt_bps(hostlvl.get('disk_read_bps')) }}</div>
<div><span style="color:var(--text-dim);">wr</span> {{ fmt_bps(hostlvl.get('disk_write_bps')) }}</div>
</div>
</div>
{% if hostlvl.get('temp_c_max') is not none %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Temp (max)</div>
{% set t = hostlvl.get('temp_c_max') %}
{{ gauge(t, 'temp', "%.0f°C"|format(t)) }}
<span class="stat-val" style="font-size:1.6rem;color:{% if t < 70 %}var(--green){% elif t < 85 %}var(--yellow){% else %}var(--red){% endif %};">{{ "%.0f"|format(t) }}°C</span>
</div>
{% endif %}
{% if hostlvl.get('psi_mem_some_avg10') is not none %}
@@ -96,7 +113,7 @@
<div style="display:flex;justify-content:space-between;font-size:0.72rem;color:var(--text-dim);margin-bottom:0.2rem;">
<span>core {{ idx }}</span><span>{% if pct is not none %}{{ "%.0f"|format(pct) }}%{% else %}—{% endif %}</span>
</div>
{{ bar(pct, 'cpu') }}
{{ bar(pct) }}
</div>
{% endfor %}
</div>
@@ -111,9 +128,9 @@
<div style="margin-bottom:0.6rem;">
<div style="display:flex;justify-content:space-between;font-size:0.8rem;margin-bottom:0.2rem;">
<span style="font-family:ui-monospace,monospace;">{{ mount }}</span>
<span style="color:var(--text-muted);">{{ m.get('disk_used_bytes')|bytes }} / {{ m.get('disk_total_bytes')|bytes }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %}</span>
<span style="color:var(--text-muted);">{{ fmt_bytes(m.get('disk_used_bytes')) }} / {{ fmt_bytes(m.get('disk_total_bytes')) }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %}</span>
</div>
{{ bar(m.get('disk_used_pct'), 'disk') }}
{{ bar(m.get('disk_used_pct')) }}
</div>
{% endfor %}
</div>
@@ -128,7 +145,7 @@
{% for iface, m in nets.items() %}
<div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
<span style="font-family:ui-monospace,monospace;">{{ iface }}</span>
<span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">↓ {{ m.get('net_rx_bps')|rate }} · ↑ {{ m.get('net_tx_bps')|rate }}</span>
<span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">↓ {{ fmt_bps(m.get('net_rx_bps')) }} · ↑ {{ fmt_bps(m.get('net_tx_bps')) }}</span>
</div>
{% endfor %}
</div>
@@ -139,7 +156,7 @@
{% for dev, m in disks_io.items() %}
<div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
<span style="font-family:ui-monospace,monospace;">{{ dev }}</span>
<span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">rd {{ m.get('disk_read_bps')|rate }} · wr {{ m.get('disk_write_bps')|rate }}</span>
<span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">rd {{ fmt_bps(m.get('disk_read_bps')) }} · wr {{ fmt_bps(m.get('disk_write_bps')) }}</span>
</div>
{% endfor %}
</div>
@@ -155,7 +172,7 @@
{% for label, c in temps.items() %}
<div style="display:flex;justify-content:space-between;gap:0.5rem;font-size:0.8rem;padding:0.15rem 0;">
<span style="white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ label }}</span>
<span class="tone-{{ status_class(c, 'temp') }}" style="font-variant-numeric:tabular-nums;">{% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %}</span>
<span style="color:{% if c is none %}var(--text-dim){% elif c < 70 %}var(--green){% elif c < 85 %}var(--yellow){% else %}var(--red){% endif %};font-variant-numeric:tabular-nums;">{% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %}</span>
</div>
{% endfor %}
</div>
plugins/host_agent/templates/_host_vitals.html
+30 -35
View File
@@ -1,40 +1,35 @@
{# Host health header — live polled into the host hub: live or stale, the worst
reading against Settings → Thresholds, last seen, then CPU / memory / disk /
load tiles graded by the same cutoffs (`grades`; CPU, memory and load within
the host's own normal for the hour read healthy, and `usual` says so). Renders nothing until the agent
reports (the Agent panel below then shows provisioning). #}
{% from "_macros.html" import status_badge, stat_tile %}
{% macro pct(v) %}{{ '%.0f%%'|format(v) if v is not none else '—' }}{% endmacro %}
{% set grades = grades or {} %}
{# Host vitals strip — compact CPU/MEM/DISK/LOAD + pressure, live-polled into the
host hub. Renders nothing until the agent reports (the Agent panel below then
shows provisioning). #}
{% if reporting %}
<section class="card host-health" aria-label="Host health">
<div class="host-health-head">
{{ status_badge('stale' if stale else 'up', 'Stale' if stale else 'Live') }}
<span class="host-health-summary">
{%- if stale %}No report since {{ reg.last_seen_at|ago }}; the readings below are from then.
{%- elif worst %}<span class="tone-{{ worst[2] }}">{{ worst[0] }} {{ pct(worst[1]) }}</span> is past its {{ "critical" if worst[2] == "crit" else "warning" }} threshold.
{%- elif usual %}Every reading is within its thresholds or its usual level for this hour.
{%- else %}Every reading is within its thresholds.{% endif -%}
{% set lbl = "font-size:0.68rem;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em;" %}
{% macro vital(label, value, text, kind, spark) %}
<div style="min-width:88px;">
<div style="{{ lbl }}">{{ label }}</div>
<div style="font-size:1.5rem;font-weight:700;line-height:1.1;{{ threshold_style(value, kind) }}">{{ text }}</div>
<div style="margin-top:0.2rem;">{{ spark | safe }}</div>
</div>
{% endmacro %}
<div class="card" style="display:flex;flex-wrap:wrap;align-items:flex-start;gap:1rem 2rem;margin-bottom:1rem;">
{{ vital("CPU", cpu, ('%.0f%%'|format(cpu) if cpu is not none else '—'), 'cpu', sparks.cpu) }}
{{ vital("Memory", mem, ('%.0f%%'|format(mem) if mem is not none else '—'), 'mem', sparks.mem) }}
{{ vital("Disk /", disk_root, ('%.0f%%'|format(disk_root) if disk_root is not none else '—'), 'disk', sparks.disk) }}
{{ vital("Load /core", load_per_core, ('%d%%'|format(load_per_core) if load_per_core is not none else '—'), 'load', sparks.load) }}
<div style="margin-left:auto;display:flex;flex-direction:column;align-items:flex-end;gap:0.3rem;font-size:0.75rem;color:var(--text-muted);text-align:right;">
<span>
{% if stale %}<span class="badge badge-red">stale</span>{% else %}<span class="badge badge-green">live</span>{% endif %}
{% if reg and reg.agent_version %}<span style="margin-left:0.3rem;">v{{ reg.agent_version }}</span>{% endif %}
</span>
<span class="problem-detail">
{%- if not stale %}seen {{ reg.last_seen_at|ago }}{% endif %}
{%- if reg.agent_version %}{{ " · " if not stale }}agent v{{ reg.agent_version }}{% endif -%}
{%- if clock and clock[0] %} · <span{% if clock[1] in ('warn', 'crit') %} class="tone-{{ clock[1] }}" title="Further from Steward's clock than Settings → Thresholds allows: check what keeps this host's time in sync"{% endif %}>{{ clock[0] }}</span>{% endif -%}
{% if psi.cpu is not none or psi.mem is not none or psi.io is not none %}
<span title="Pressure stall — % of the last 10s tasks waited for the resource">
<span style="color:var(--text-dim);text-transform:uppercase;font-size:0.66rem;letter-spacing:0.04em;">Pressure 10s</span>
cpu {{ '%.0f%%'|format(psi.cpu) if psi.cpu is not none else '—' }}
· mem {{ '%.0f%%'|format(psi.mem) if psi.mem is not none else '—' }}
· io {{ '%.0f%%'|format(psi.io) if psi.io is not none else '—' }}
</span>
{% endif %}
{% if reg and reg.last_seen_at %}<span>seen {{ reg.last_seen_at.strftime("%H:%M:%S") }} UTC</span>{% endif %}
</div>
<div class="stat-strip host-health-tiles" style="margin-bottom:0;">
{{ stat_tile("CPU", pct(cpu), state=(grades.cpu or [status_class(cpu, 'cpu')])[0], hint=sparks.cpu|safe) }}
{{ stat_tile("Memory", pct(mem), state=(grades.mem or [status_class(mem, 'mem')])[0], hint=sparks.mem|safe) }}
{{ stat_tile("Disk /", pct(disk_root), state=(grades.disk or [status_class(disk_root, 'disk')])[0], hint=sparks.disk|safe) }}
{{ stat_tile("Load per core", pct(load_per_core), state=(grades.load or [status_class(load_per_core, 'load')])[0], hint=sparks.load|safe) }}
</div>
{% if usual and not stale %}
<p class="field-hint" style="margin-top:0.6rem;">{{ usual | join("; ") | capitalize }}.</p>
{% endif %}
{% if psi.cpu is not none or psi.mem is not none or psi.io is not none %}
<p class="field-hint" style="margin-top:0.6rem;" title="Pressure stall: the share of the last 10 s that tasks waited for the resource">
Pressure (10 s): cpu {{ pct(psi.cpu) }} · mem {{ pct(psi.mem) }} · io {{ pct(psi.io) }}
</p>
{% endif %}
</section>
</div>
{% endif %}
plugins/host_agent/templates/host_detail.html
+13 -13
View File
@@ -54,30 +54,30 @@
interaction: { mode: "index", intersect: false },
elements: { point: { radius: 0 } },
scales: {
x: { type: "linear", ticks: { callback: (v) => fmtTime(v), maxTicksLimit: 8, font: { size: 10 } } },
y: { beginAtZero: true, max: ymax, ticks: { font: { size: 10 } } },
x: { type: "linear", ticks: { callback: (v) => fmtTime(v), maxTicksLimit: 8, color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
y: { beginAtZero: true, max: ymax, ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
},
plugins: {
legend: { labels: { boxWidth: 10, font: { size: 11 } } },
legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } },
tooltip: { callbacks: { title: (items) => items.length ? fmtTime(items[0].parsed.x) : "" } },
},
});
const defs = {
"chart-util": { ymax: 100, ds: [
{ key: "cpu_pct", label: "CPU %", color: stewardColor("--gold") },
{ key: "mem_used_pct", label: "Mem %", color: stewardColor("--green") },
{ key: "disk_used_pct_worst", label: "Disk % (worst)", color: stewardColor("--yellow") },
{ key: "cpu_pct", label: "CPU %", color: "#c8a840" },
{ key: "mem_used_pct", label: "Mem %", color: "#4aa86a" },
{ key: "disk_used_pct_worst", label: "Disk % (worst)", color: "#c87840" },
] },
"chart-net": { ymax: undefined, ds: [
{ key: "net_rx_bps", label: "Net RX", color: stewardColor("--green") },
{ key: "net_tx_bps", label: "Net TX", color: stewardColor("--red") },
{ key: "disk_read_bps", label: "Disk read", color: stewardColor("--blue") },
{ key: "disk_write_bps", label: "Disk write", color: stewardColor("--gold") },
{ key: "net_rx_bps", label: "Net RX", color: "#4aa86a" },
{ key: "net_tx_bps", label: "Net TX", color: "#c84048" },
{ key: "disk_read_bps", label: "Disk read", color: "#6aa0d0" },
{ key: "disk_write_bps", label: "Disk write", color: "#c8a840" },
] },
"chart-pressure": { ymax: undefined, ds: [
{ key: "load_1m", label: "Load 1m", color: stewardColor("--gold") },
{ key: "psi_mem_some_avg10", label: "Mem PSI %", color: stewardColor("--red") },
{ key: "temp_c_max", label: "Temp °C", color: stewardColor("--yellow") },
{ key: "load_1m", label: "Load 1m", color: "#c8a840" },
{ key: "psi_mem_some_avg10", label: "Mem PSI %", color: "#c84048" },
{ key: "temp_c_max", label: "Temp °C", color: "#c87840" },
] },
};
const charts = {};
plugins/host_agent/templates/install.sh.j2
-4
View File
@@ -45,10 +45,6 @@ cat > "$CONF_FILE" <<EOF
url = $STEWARD_URL
token = $AGENT_TOKEN
interval_seconds = 30
# Container logs are collected by default. To opt this host out entirely:
# docker_logs_enabled = false
# To skip specific noisy containers (comma-separated names):
# docker_log_exclude = watchtower, some-chatty-service
EOF
chown "root:$AGENT_USER" "$CONF_FILE"
chmod 0640 "$CONF_FILE"
plugins/host_agent/templates/panel.html
+51 -65
View File
@@ -4,9 +4,9 @@
<div style="display:flex;align-items:baseline;justify-content:space-between;margin-bottom:0.75rem;gap:1rem;flex-wrap:wrap;">
<h3 class="section-title" style="margin-bottom:0;">Agent</h3>
{% if reporting %}
<span style="font-size:0.78rem;" class="tone-{{ 'warn' if stale else 'ok' }}">
<span style="font-size:0.78rem;color:{{ 'var(--yellow)' if stale else 'var(--green)' }};">
{{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %}
· last seen {{ reg.last_seen_at|ago }}
· last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }}
</span>
{% elif reg %}
<span style="font-size:0.78rem;color:var(--text-muted);">pending — no check-in yet</span>
@@ -15,57 +15,52 @@
{% if reporting %}
{# Reporting: live vitals are shown by the vitals strip above; this panel is
the way to the full metrics, with the lifecycle actions folded under
"Manage agent" (#5278). #}
<a href="/plugins/host_agent/{{ host.id }}/" class="btn btn-sm btn-ghost">Full metrics →</a>
lifecycle/management only. #}
<div style="display:flex;gap:0.5rem;flex-wrap:wrap;align-items:center;">
<a href="/plugins/host_agent/{{ host.id }}/" class="btn btn-sm btn-ghost">Full metrics →</a>
{% if session.user_role == 'admin' %}
{% if ansible_available and target %}
<form method="post" action="/plugins/host_agent/update" style="margin:0;">
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<button type="submit" class="btn btn-sm" title="Refresh agent.py + restart (token preserved)">Update agent</button>
</form>
{% endif %}
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/rotate-token" style="margin:0;"
onsubmit="return confirm('Rotate token? The agent stops reporting until reinstalled/updated with the new token.');">
<button type="submit" class="btn btn-sm btn-ghost">Rotate token</button>
</form>
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0;"
onsubmit="return confirm('Remove this agent registration? Metrics stop until re-registered.');">
<button type="submit" class="btn btn-sm btn-danger">Remove</button>
</form>
{% endif %}
</div>
{% if ansible_available and not target %}
<p style="color:var(--text-dim);font-size:0.8rem;margin:0.6rem 0 0;">
Link an Ansible target (above) to enable one-click agent updates.
</p>
{% endif %}
{% if session.user_role == 'admin' %}
{% if session.user_role == 'admin' and ansible_available and target and managed_key_set %}
<details style="margin-top:0.7rem;">
<summary style="cursor:pointer;font-size:0.82rem;color:var(--text-muted);">Manage agent</summary>
<div style="display:flex;gap:0.5rem;flex-wrap:wrap;align-items:center;margin-top:0.6rem;">
{% if ansible_available and target %}
<form method="post" action="/plugins/host_agent/update" style="margin:0;">{{ csrf_field() }}
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<button type="submit" class="btn btn-sm" title="Refresh agent.py and restart it; the token is kept">Update agent</button>
</form>
{% endif %}
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/rotate-token" style="margin:0;"
data-confirm="Rotate token? The agent stops reporting until reinstalled/updated with the new token.">{{ csrf_field() }}
<button type="submit" class="btn btn-sm btn-ghost">Rotate token</button>
</form>
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0;"
data-confirm="Unregister this host's agent? Its token stops working and metrics stop until it is registered again. The agent itself stays installed on the host.">{{ csrf_field() }}
<button type="submit" class="btn btn-sm btn-danger">Unregister agent</button>
</form>
</div>
{% if ansible_available and not target %}
<p style="color:var(--text-dim);font-size:0.8rem;margin:0.6rem 0 0;">
Link an Ansible target (below) to update the agent from here.
<summary style="cursor:pointer;font-size:0.8rem;color:var(--text-muted);">Re-provision (reinstall the steward account + managed key, then the agent)</summary>
<p style="font-size:0.78rem;color:var(--text-dim);margin:0.5rem 0;">
Use after regenerating the managed key, or if SSH auth as <code>steward</code> breaks.
Connects over a one-time bootstrap user + password (not stored).
</p>
{% endif %}
{% if ansible_available and target and managed_key_set %}
<details style="margin-top:0.7rem;">
<summary style="cursor:pointer;font-size:0.8rem;color:var(--text-muted);">Re-provision (reinstall the steward account + managed key, then the agent)</summary>
<p style="font-size:0.78rem;color:var(--text-dim);margin:0.5rem 0;">
Use after regenerating the managed key, or if SSH auth as <code>steward</code> breaks.
Connects over a one-time bootstrap user + password (not stored).
</p>
<form method="post" action="/plugins/host_agent/provision"
style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<div class="form-group" style="margin:0;">
<label>Bootstrap user</label>
<input type="text" name="bootstrap_user" required placeholder="root" style="width:8rem;" autocomplete="off">
</div>
<div class="form-group" style="margin:0;">
<label>Bootstrap password</label>
<input type="password" name="bootstrap_password" required style="width:10rem;" autocomplete="new-password">
</div>
<button type="submit" class="btn btn-sm">Re-provision</button>
</form>
</details>
{% endif %}
<form method="post" action="/plugins/host_agent/provision"
style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;">
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<div class="form-group" style="margin:0;">
<label>Bootstrap user</label>
<input type="text" name="bootstrap_user" required placeholder="root" style="width:8rem;" autocomplete="off">
</div>
<div class="form-group" style="margin:0;">
<label>Bootstrap password</label>
<input type="password" name="bootstrap_password" required style="width:10rem;" autocomplete="new-password">
</div>
<button type="submit" class="btn btn-sm">Re-provision</button>
</form>
</details>
{% endif %}
@@ -91,7 +86,8 @@
{% elif not ansible_available %}
<p style="color:var(--text-dim);font-size:0.85rem;">
Ansible isn't available, so the agent can't be deployed from here. Install it manually instead (below).
Ansible isn't available, so the agent can't be deployed from here. Install it manually with the
<a href="/plugins/host_agent/fleet/">curl install command</a>.
</p>
{% elif not managed_key_set %}
@@ -100,7 +96,7 @@
border-radius:6px;padding:0.7rem 0.9rem;display:flex;align-items:center;gap:0.8rem;flex-wrap:wrap;">
<span style="color:var(--yellow);">⚠</span>
<span style="font-size:0.84rem;flex:1;min-width:13rem;">No managed SSH key yet — Steward needs one to log into hosts.</span>
<form method="post" action="/settings/ansible/generate-key" style="margin:0;">{{ csrf_field() }}
<form method="post" action="/settings/ansible/generate-key" style="margin:0;">
<input type="hidden" name="next" value="/hosts/{{ host.id }}">
<button type="submit" class="btn btn-sm">Generate managed key</button>
</form>
@@ -119,7 +115,7 @@
</p>
{# Provision: brand-new host (creates steward account + key over a one-time password) #}
<form method="post" action="/plugins/host_agent/provision"
style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;margin-bottom:0.75rem;">{{ csrf_field() }}
style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;margin-bottom:0.75rem;">
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<div style="font-size:0.78rem;color:var(--text-muted);width:100%;">Provision (first contact — fresh host):</div>
<div class="form-group" style="margin:0;">
@@ -133,26 +129,16 @@
<button type="submit" class="btn btn-sm">Provision</button>
</form>
{# Install: host already has the steward account (managed key works) #}
<form method="post" action="/plugins/host_agent/deploy" style="display:flex;gap:0.6rem;align-items:center;">{{ csrf_field() }}
<form method="post" action="/plugins/host_agent/deploy" style="display:flex;gap:0.6rem;align-items:center;">
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<span style="font-size:0.78rem;color:var(--text-muted);">Already provisioned?</span>
<button type="submit" class="btn btn-sm btn-ghost">Install agent (managed key)</button>
</form>
{% endif %}
{% if not reg and session.user_role == 'admin' %}
{# A manual install: no Ansible, no SSH. Issues this host's token and shows
the one-line install command on the fleet page, once (#5282). #}
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/register"
style="display:flex;gap:0.6rem;align-items:center;flex-wrap:wrap;margin:0.75rem 0 0;">{{ csrf_field() }}
<span style="font-size:0.78rem;color:var(--text-muted);">Or by hand on the host:</span>
<button type="submit" class="btn btn-sm btn-ghost">Install manually</button>
</form>
{% endif %}
{% if reg and session.user_role == 'admin' %}
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0.75rem 0 0;"
data-confirm="Clear this pending registration? Its token is discarded.">{{ csrf_field() }}
onsubmit="return confirm('Clear this pending registration? Its token is discarded.');">
<button type="submit" class="btn btn-sm btn-ghost">Clear pending registration</button>
</form>
{% endif %}
plugins/host_agent/templates/settings_list.html
+57 -47
View File
@@ -1,5 +1,5 @@
{% extends "base.html" %}
{% from "_macros.html" import crumbs, status_badge %}
{% from "_macros.html" import crumbs %}
{% block title %}Agent fleet — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), ("Agent fleet", "")]) }}{% endblock %}
{% block content %}
@@ -8,12 +8,11 @@
</div>
<p style="color:var(--text-muted);font-size:0.84rem;margin:-0.4rem 0 1.25rem;">
Bulk agent operations across your inventory. For a single host, use its
<a href="/hosts/">host page</a>: provisioning, a manual install command and
metrics live there. New hosts are added with <a href="/hosts/?add=1">Add host</a>.
<a href="/hosts/">host page</a> — provisioning and metrics live there.
</p>
{% if new_token %}
<div class="card" style="background:var(--fs-accent-faint);">
<div class="card" style="background:var(--accent-bg);">
<h3>New token — copy this install command now</h3>
<p style="font-size:0.82rem;color:var(--text-muted);">
This is the only time the raw token is shown. Rotate if you lose it.
@@ -22,43 +21,19 @@
</div>
{% endif %}
<div class="card-flush" style="margin-bottom:1.5rem;">
<div class="table-wrap">
<table class="table">
<thead>
<tr>
<th>Status</th><th>Host</th><th>Last seen</th><th>Agent version</th><th>Reported IP</th><th>Distro</th>
<th class="td-actions">Actions</th>
</tr>
</thead>
<tbody>
{% for item in registrations %}
<tr>
<td>{{ status_badge(item.state, {'up': 'Reporting', 'stale': 'Silent', 'pending': 'Never reported'}[item.state]) }}</td>
<td>{% if item.host %}<a href="/hosts/{{ item.host.id }}">{{ item.host.name }}</a>{% else %}<span class="mono">{{ item.reg.host_id }}</span>{% endif %}</td>
<td>{% if item.reg.last_seen_at %}<span title="{{ item.reg.last_seen_at.strftime('%Y-%m-%d %H:%M:%S UTC') }}">{{ item.reg.last_seen_at|ago }}</span>{% else %}never{% endif %}</td>
<td>{{ item.reg.agent_version or "—" }}</td>
<td class="mono">{{ item.reg.host_ip or "—" }}</td>
<td>{{ item.reg.distro or "—" }}</td>
<td class="td-actions">
<form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/rotate-token"
style="display:inline;"
data-confirm="Rotate the token for {{ item.name }}? The existing agent will stop working until reinstalled with the new token.">{{ csrf_field() }}
<button type="submit" class="btn btn-ghost btn-sm">Rotate token</button>
</form>
<form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/delete"
style="display:inline;"
data-confirm="Unregister the agent for {{ item.name }}? Its token stops working and metrics stop until it is registered again. The agent itself stays installed on the host.">{{ csrf_field() }}
<button type="submit" class="btn btn-danger btn-sm">Unregister</button>
</form>
</td>
</tr>
{% else %}
<tr><td colspan="7" class="empty">No agents registered yet. <a href="/hosts/?add=1">Add a host</a>, then install its agent from the host's page.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
<div class="card">
<form method="post" action="/plugins/host_agent/fleet/add-host"
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
<div class="form-group" style="margin-bottom:0;">
<label>Host name</label>
<input type="text" name="name" required>
</div>
<div class="form-group" style="margin-bottom:0;">
<label>Address (optional)</label>
<input type="text" name="address">
</div>
<button type="submit" class="btn">Add host</button>
</form>
</div>
{% if ansible_available %}
@@ -74,7 +49,7 @@
<h3 style="margin-bottom:0.4rem;">Agent lifecycle via Ansible</h3>
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.6rem;">
These actions run bundled <strong>Ansible playbooks</strong> to deploy and maintain the
Steward monitoring agent on your <a href="/ansible/inventory/">inventory hosts</a>.
Steward monitoring agent on your <a href="/ansible/inventory/targets">inventory hosts</a>.
Steward generates each host's API token automatically and connects over SSH as the managed
<code>steward</code> account — you'll be dropped into the live Ansible run to watch it.
</p>
@@ -88,14 +63,14 @@
<span style="font-size:0.84rem;flex:1;min-width:14rem;">
No managed SSH key yet — Steward needs one to log into hosts as <code>steward</code>.
</span>
<form method="post" action="/settings/ansible/generate-key" style="margin:0;">{{ csrf_field() }}
<form method="post" action="/settings/ansible/generate-key" style="margin:0;">
<input type="hidden" name="next" value="/plugins/host_agent/fleet/">
<button type="submit" class="btn btn-sm">Generate managed key</button>
</form>
</div>
{% elif not (deploy_targets or deploy_groups) %}
<p style="font-size:0.85rem;color:var(--text-dim);margin:0;">
No Ansible inventory targets yet. Add some under <a href="/ansible/inventory/">Ansible → Inventory</a>.
No Ansible inventory targets yet. Add some under <a href="/ansible/inventory/targets">Ansible → Inventory</a>.
</p>
{% endif %}
</div>
@@ -110,7 +85,7 @@
agent. Run this once per host.
</p>
<form method="post" action="/plugins/host_agent/provision"
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
<div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
<div class="form-group" style="margin-bottom:0;">
<label>Bootstrap user</label>
@@ -136,7 +111,7 @@
mints a fresh token, and installs the agent. No <code>curl | sh</code> needed.
</p>
<form method="post" action="/plugins/host_agent/deploy"
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
<div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
<div class="form-group" style="margin-bottom:0;">
<label>Report interval (s)</label>
@@ -154,7 +129,7 @@
so identity is preserved.
</p>
<form method="post" action="/plugins/host_agent/update"
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
<div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
<button type="submit" class="btn">Update</button>
</form>
@@ -162,4 +137,39 @@
{% endif %}
{% endif %}
<div class="card-flush">
<table class="table">
<thead>
<tr>
<th>Host</th><th>Reported IP</th><th>Agent version</th><th>Distro</th>
<th>Last seen</th><th class="td-actions">Actions</th>
</tr>
</thead>
<tbody>
{% for item in registrations %}
<tr>
<td>{{ item.host.name if item.host else item.reg.host_id }}</td>
<td>{{ item.reg.host_ip or "—" }}</td>
<td>{{ item.reg.agent_version or "—" }}</td>
<td>{{ item.reg.distro or "—" }}</td>
<td>{{ item.reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") if item.reg.last_seen_at else "never" }}</td>
<td class="td-actions">
<form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/rotate-token"
style="display:inline;"
onsubmit="return confirm('Rotate the token for this host? The existing agent will stop working until reinstalled with the new token.');">
<button type="submit" class="btn btn-ghost btn-sm">Rotate token</button>
</form>
<form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/delete"
style="display:inline;"
onsubmit="return confirm('Delete this host registration? The agent will be unable to report metrics until re-registered.');">
<button type="submit" class="btn btn-danger btn-sm">Delete</button>
</form>
</td>
</tr>
{% else %}
<tr><td colspan="6" class="empty">No hosts registered. Add one above.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
{% endblock %}
plugins/host_agent/templates/widget_history.html
+15 -17
View File
@@ -16,14 +16,16 @@
toggle that re-requests this widget without entering edit mode. #}
<div style="flex:0 0 auto;display:flex;align-items:center;gap:0.5rem;flex-wrap:wrap;font-size:0.75rem;color:var(--text-muted);margin-bottom:0.3rem;">
{% if session.user_id %}
<a href="/hosts/{{ host.id }}" style="font-weight:500;">{{ host.name }}</a>
<a href="/hosts/{{ host.id }}" style="font-weight:600;">{{ host.name }}</a>
{% else %}
<strong style="color:var(--text);">{{ host.name }}</strong>
{% endif %}
<span class="hist-range" style="margin-left:auto;display:inline-flex;gap:0.25rem;">
<span class="hist-range" style="margin-left:auto;display:inline-flex;border:1px solid var(--border-mid);border-radius:5px;overflow:hidden;">
{% for opt in [1, 6, 24] %}
<button type="button" onclick="histRange({{ wid|int }}, {{ opt|int }})"
class="range-btn{% if opt == hours %} active{% endif %}">{{ opt }}h</button>
<button type="button" onclick="histRange({{ wid }}, '{{ host.id }}', {{ opt }})"
style="border:0;cursor:pointer;padding:0.1rem 0.45rem;font-size:0.72rem;
background:{% if opt == hours %}var(--accent){% else %}transparent{% endif %};
color:{% if opt == hours %}#fff{% else %}var(--text-muted){% endif %};">{{ opt }}h</button>
{% endfor %}
</span>
</div>
@@ -34,15 +36,11 @@
<script>
// Live time-range switch. Rewrites the parent cell's hx-get so the choice sticks
// across polls (htmx re-reads the attribute each poll), then fetches immediately.
// Only `hours` changes: the rest of the query (host, widget id, a share link's
// token) is kept, so the switch also works on a shared dashboard.
window.histRange = function (wid, hours) {
window.histRange = function (wid, hostId, hours) {
var cell = document.getElementById("widget-" + wid);
if (!cell) return;
var parts = (cell.getAttribute("hx-get") || "").split("?");
var query = new URLSearchParams(parts[1] || "");
query.set("hours", hours);
var url = parts[0] + "?" + query.toString();
var base = (cell.getAttribute("hx-get") || "").split("?")[0];
var url = base + "?host_id=" + encodeURIComponent(hostId) + "&hours=" + hours + "&wid=" + wid;
cell.setAttribute("hx-get", url);
if (window.htmx) window.htmx.ajax("GET", url, { target: cell, swap: "innerHTML" });
};
@@ -55,9 +53,9 @@ window.histRange = function (wid, hours) {
return ("0" + d.getHours()).slice(-2) + ":" + ("0" + d.getMinutes()).slice(-2);
};
var ds = [
{ key: "cpu_pct", label: "CPU %", color: stewardColor("--gold") },
{ key: "mem_used_pct", label: "Mem %", color: stewardColor("--green") },
{ key: "disk_root", label: "Disk / %", color: stewardColor("--yellow") },
{ key: "cpu_pct", label: "CPU %", color: "#c8a840" },
{ key: "mem_used_pct", label: "Mem %", color: "#4aa86a" },
{ key: "disk_root", label: "Disk / %", color: "#c87840" },
];
new Chart(el.getContext("2d"), {
type: "line",
@@ -73,14 +71,14 @@ window.histRange = function (wid, hours) {
interaction: { mode: "index", intersect: false },
elements: { point: { radius: 0 } },
scales: {
x: { type: "linear", ticks: { callback: function (v) { return fmtTime(v); }, maxTicksLimit: 6, font: { size: 10 } } },
x: { type: "linear", ticks: { callback: function (v) { return fmtTime(v); }, maxTicksLimit: 6, color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
// Fixed 0–{{ y_max }} ceiling (server snaps the data peak up to the next
// 10% band). Steady across refreshes — auto-scaling made the zoom jump on
// every poll — while a banded ceiling still keeps the lines filling the panel.
y: { min: 0, max: {{ y_max }}, ticks: { font: { size: 10 } } },
y: { min: 0, max: {{ y_max }}, ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
},
plugins: {
legend: { labels: { boxWidth: 10, font: { size: 11 } } },
legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } },
tooltip: { callbacks: { title: function (items) { return items.length ? fmtTime(items[0].parsed.x) : ""; } } },
},
},
plugins/host_agent/templates/widget_table.html
+77
View File
@@ -0,0 +1,77 @@
{# host_agent widget — fleet glance. One horizontal row per host, zebra-striped
so adjacent hosts read as distinct groupings: health dot + host name on the
left, metric cells (cpu/mem/disk/load) each with a trend sparkline laid out
horizontally on the right. The name wraps (never hard-truncated) but the row
stays horizontal. #}
{% if rows %}
{% macro metric_cell(label, value, fmt, svg, style="", title="") %}
<div style="min-width:62px;flex:0 0 auto;" title="{{ title }}">
<div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
<div style="font-weight:600;font-size:0.8rem;font-variant-numeric:tabular-nums;{{ style }}">
{% if value is not none %}{{ fmt|format(value) }}{% else %}—{% endif %}
</div>
<div style="line-height:0;height:16px;">{{ svg | safe }}</div>
</div>
{% endmacro %}
{# Human-readable throughput, matching the host-detail page. #}
{% macro fmt_bps(v) %}
{%- if v is none -%}—
{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s
{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s
{%- else -%}{{ "%.0f"|format(v) }} B/s
{%- endif -%}
{% endmacro %}
{# Two-line down/up (or read/write) throughput cell. #}
{% macro io_cell(label, down_sym, down_val, up_sym, up_val, title="") %}
<div style="min-width:80px;flex:0 0 auto;" title="{{ title }}">
<div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
<div style="font-size:0.72rem;font-variant-numeric:tabular-nums;line-height:1.3;">
<div>{{ down_sym }} {{ fmt_bps(down_val) }}</div>
<div>{{ up_sym }} {{ fmt_bps(up_val) }}</div>
</div>
</div>
{% endmacro %}
<div class="host-fleet">
{% for r in rows %}
{% set stale = r.stale %}
<div style="display:flex;align-items:center;gap:0.6rem 1rem;flex-wrap:wrap;padding:0.5rem 0.6rem;border-radius:4px;{% if loop.index0 % 2 %}background:var(--bg-elevated);{% endif %}">
{# Left — health dot + full host name (wraps, never truncated) + last seen #}
<div style="display:flex;align-items:center;gap:0.5rem;flex:1 1 150px;min-width:120px;">
<span class="dot {% if stale %}dot-dim{% elif (r.cpu_pct or 0) >= 90 or (r.mem_used_pct or 0) >= 90 or (r.disk_worst or 0) >= 90 %}dot-warn{% else %}dot-up{% endif %}"
style="flex-shrink:0;"
title="{% if stale %}stale / no recent data{% else %}warns at CPU/memory ≥90% or any disk mount ≥90% (the 'disk /' figure shows root only){% endif %}"></span>
<span style="min-width:0;">
{% if session.user_id %}
<a href="/hosts/{{ r.host.id }}" style="font-weight:600;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</a>
{% else %}
<span style="font-weight:600;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</span>
{% endif %}
<span style="display:block;font-size:0.7rem;color:var(--text-dim);">
{{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }}
</span>
</span>
</div>
{# Right — metric cells with value + trend, laid out horizontally #}
<div style="display:flex;flex-wrap:wrap;gap:0.4rem 1rem;justify-content:flex-end;">
{{ metric_cell("cpu", r.cpu_pct, "%.0f%%", r.sparks.cpu, threshold_style(r.cpu_pct, 'cpu'), "Average CPU utilization") }}
{{ metric_cell("mem", r.mem_used_pct, "%.0f%%", r.sparks.mem, threshold_style(r.mem_used_pct, 'mem'), "Memory in use") }}
{{ metric_cell("disk /", r.disk_root, "%.0f%%", r.sparks.disk, threshold_style(r.disk_root, 'disk'), "Root filesystem (/) usage") }}
{{ metric_cell("load", r.load_1m, "%.2f", r.sparks.load, "", "Load average (1m)") }}
{# Extra agent metrics — shown only when the host reports them (VMs/containers
often have no temp sensor, etc.) so absent data doesn't clutter the row. #}
{% if r.net_rx_bps is not none or r.net_tx_bps is not none %}
{{ io_cell("net", "↓", r.net_rx_bps, "↑", r.net_tx_bps, "Network throughput (down / up)") }}
{% endif %}
{% if r.disk_read_bps is not none or r.disk_write_bps is not none %}
{{ io_cell("disk i/o", "rd", r.disk_read_bps, "wr", r.disk_write_bps, "Disk I/O (read / write)") }}
{% endif %}
{% if r.temp_max is not none %}
{{ metric_cell("temp", r.temp_max, "%.0f°C", "", threshold_style(r.temp_max, 'temp'), "Hottest sensor") }}
{% endif %}
</div>
</div>
{% endfor %}
</div>
{% else %}
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No hosts with agent data yet.</p>
{% endif %}
plugins/index.yaml
+74
View File
@@ -0,0 +1,74 @@
# steward-plugins / index.yaml
#
# Plugin catalog for Steward.
# Fetched by the app at: Settings → Plugins → Plugin Catalog
#
# After updating an entry, commit and push — changes are live within 5 minutes
# (the app caches this file in-process for 300 seconds).
#
# See docs/plugins/index.yaml.example in the main app repo for the full schema.
version: 1
updated: "2026-03-22"
plugins:
- name: docker
version: "2.0.0"
description: "Per-host Docker container status + resource usage, collected by the Steward host agent"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v2.0.0/docker.zip"
checksum_sha256: ""
tags:
- containers
- docker
- infrastructure
- name: traefik
version: "1.0.0"
description: "Traefik reverse proxy metrics and access log integration"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip"
checksum_sha256: ""
tags:
- proxy
- metrics
- access-log
- name: unifi
version: "1.0.0"
description: "UniFi Network controller integration — WAN health, devices, clients, DPI"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/unifi-v1.0.0/unifi.zip"
checksum_sha256: ""
tags:
- network
- unifi
- ubiquiti
- name: ups
version: "1.0.0"
description: "UPS monitoring via NUT (Network UPS Tools) with Ansible shutdown automation"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/ups"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/ups-v1.0.0/ups.zip"
checksum_sha256: ""
tags:
- ups
- power
- nut
plugins/prometheus/__init__.py
-70
View File
@@ -1,70 +0,0 @@
# plugins/prometheus/__init__.py
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from quart import Quart
_app: "Quart | None" = None
def setup(app: "Quart") -> None:
global _app
_app = app
from .models import PrometheusTarget, PrometheusWidgetTemplate # noqa: F401 — registers the tables
register_widgets()
def register_widgets() -> None:
"""One dashboard widget for every target and template (System 181's
ruling). Its target and template lists are read when the widget picker
opens, so a new target or template appears without a restart."""
from steward.core.widgets import register_widget
from . import library
register_widget({
"key": "prometheus",
"label": "Prometheus",
"description": "One scrape target drawn by a widget template: Fabled Archives, or one you copy in",
"hx_url": "/plugins/prometheus/widget",
"detail_url": "/plugins/prometheus/targets/{target}",
"plugin": "prometheus",
"poll": True,
"default_title": library.default_title,
"params": [
{"key": "target", "label": "Target", "type": "select", "default": "",
"options_from": library.target_options,
"empty": "No targets yet: add one under Prometheus first."},
{"key": "template", "label": "Template", "type": "select", "default": "",
"options_from": library.template_options,
"empty": "No templates yet."},
],
})
def get_scheduled_tasks() -> list:
from .scheduler import make_scrape_task
return [make_scrape_task(_app)]
def get_blueprint():
from .routes import prometheus_bp
return prometheus_bp
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (scraped endpoints).
return [{"label": "Prometheus", "href": "/plugins/prometheus/"}]
def get_alert_metrics() -> dict[str, list[str]]:
"""The metric names are whatever each endpoint exposes, so the list is
empty here and the rule form offers the names recorded so far."""
return {"prometheus": []}
def get_audit_entities() -> dict:
"""Audit rows about a target or a widget template name it and link to its page."""
from .models import PrometheusTarget, PrometheusWidgetTemplate
return {"prometheus_target": (PrometheusTarget, "/plugins/prometheus/targets/{}"),
"prometheus_template": (PrometheusWidgetTemplate, "/plugins/prometheus/templates/custom:{}")}
plugins/prometheus/library.py
-140
View File
@@ -1,140 +0,0 @@
"""The widget templates a Prometheus widget can draw with.
Bundled templates ship in plugins/prometheus/widgets/*.yaml, read once and
read-only; their key is "bundled:<file name>". Templates an admin writes or
copies in are rows (PrometheusWidgetTemplate, #5692); their key is
"custom:<id>". Steward fetches none from anywhere (System 181's ruling).
"""
from __future__ import annotations
import json
import re
from dataclasses import dataclass, field
from datetime import datetime
from functools import lru_cache
from pathlib import Path
from sqlalchemy import select
from .models import PrometheusTarget, PrometheusWidgetTemplate
from .template_format import Problem, Template, TemplateError, check
BUNDLED = Path(__file__).parent / "widgets"
CUSTOM = "custom:"
@dataclass
class Entry:
key: str
name: str
bundled: bool
text: str
template: Template | None = None
problems: list[Problem] = field(default_factory=list)
id: str | None = None # a custom template's row
updated_at: datetime | None = None
def _entry(key: str, fallback_name: str, text: str, bundled: bool, **row) -> Entry:
try:
template = check(text)
except TemplateError as exc:
return Entry(key, fallback_name, bundled, text, None, exc.problems, **row)
return Entry(key, template.name, bundled, text, template, **row)
@lru_cache(maxsize=1)
def bundled() -> tuple[Entry, ...]:
return tuple(_entry(f"bundled:{p.stem}", p.stem, p.read_text(encoding="utf-8"), True)
for p in sorted(BUNDLED.glob("*.yaml")))
def custom_entry(row: PrometheusWidgetTemplate) -> Entry:
# Saved only once it passed, but a later Steward may read the format
# differently: it is checked again, and says so if it no longer passes.
return _entry(CUSTOM + row.id, row.name, row.text, False, id=row.id, updated_at=row.updated_at)
async def entries(db) -> list[Entry]:
"""Every template, by name."""
rows = (await db.execute(select(PrometheusWidgetTemplate))).scalars().all()
return sorted([*bundled(), *(custom_entry(r) for r in rows)], key=lambda e: e.name.lower())
async def find(db, key: str) -> Entry | None:
if key.startswith(CUSTOM):
row = await db.get(PrometheusWidgetTemplate, key[len(CUSTOM):])
return custom_entry(row) if row else None
return next((e for e in bundled() if e.key == key), None)
async def name_taken(db, name: str, other_than: str | None = None) -> bool:
"""Another saved template has this name; the widget picker lists by name."""
q = select(PrometheusWidgetTemplate.id).where(PrometheusWidgetTemplate.name == name)
if other_than:
q = q.where(PrometheusWidgetTemplate.id != other_than)
return (await db.execute(q)).first() is not None
async def used_by(db) -> dict[str, int]:
"""How many dashboard widgets draw with each template key."""
from steward.models.dashboard import DashboardWidget
counts: dict[str, int] = {}
rows = (await db.execute(select(DashboardWidget.config_json)
.where(DashboardWidget.widget_key == "prometheus"))).scalars()
for raw in rows:
try:
key = (json.loads(raw or "{}") or {}).get("template")
except (ValueError, AttributeError):
continue
if key:
counts[key] = counts.get(key, 0) + 1
return counts
# ── Copying ──────────────────────────────────────────────────────────────────
_NAME_LINE = re.compile(r"^name:[^\n]*$", re.MULTILINE)
def renamed(text: str, name: str) -> str:
"""`text` with its top-level `name:` set to `name`, the rest as written
(its comments and layout kept)."""
line = "name: " + json.dumps(name, ensure_ascii=False)
if _NAME_LINE.search(text):
return _NAME_LINE.sub(lambda _: line, text, count=1)
return f"{line}\n{text}"
def copy_name(name: str, taken: set[str]) -> str:
""""Fabled Archives (copy)", then "(copy 2)", … a name nobody has."""
for n in range(1, 1000):
candidate = f"{name} (copy)" if n == 1 else f"{name} (copy {n})"
if candidate not in taken:
return candidate[:80]
return name[:80]
def file_name(name: str) -> str:
"""A download's file name: "Fabled Archives" → fabled-archives.yaml."""
slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
return f"{slug or 'template'}.yaml"
# ── The widget's settings ────────────────────────────────────────────────────
async def target_options(db) -> list[dict]:
rows = (await db.execute(select(PrometheusTarget.id, PrometheusTarget.name)
.order_by(PrometheusTarget.name))).all()
return [{"value": tid, "label": name} for tid, name in rows]
async def template_options(db) -> list[dict]:
return [{"value": e.key, "label": f"{e.name} (built in)" if e.bundled else e.name}
for e in await entries(db) if e.template is not None]
async def default_title(db, config: dict) -> str | None:
"""A new widget is named after its template: "Fabled Archives"."""
entry = await find(db, str(config.get("template") or ""))
return entry.name if entry and entry.template else None
plugins/prometheus/migrations/__init__.py
View File
plugins/prometheus/migrations/versions/__init__.py
View File
plugins/prometheus/migrations/versions/prometheus_001_targets.py
-46
View File
@@ -1,46 +0,0 @@
"""Prometheus scrape targets are rows edited on the Prometheus page
Each row is a /metrics endpoint: where it is, its bearer token (encrypted),
how often to scrape it, which series to keep, and the newest scrape's
outcome (milestone 506).
Revision ID: prometheus_001_targets
Revises: (none — a branch off core via depends_on)
Create Date: 2026-10-12
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "prometheus_001_targets"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = "prometheus"
depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
def upgrade() -> None:
op.create_table(
"prometheus_targets",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("name", sa.String(100), nullable=False, unique=True),
sa.Column("url", sa.String(500), nullable=False),
sa.Column("token", sa.Text, nullable=False, server_default=""),
sa.Column("verify_tls", sa.Boolean, nullable=False, server_default=sa.true()),
sa.Column("interval_seconds", sa.Integer, nullable=False, server_default="60"),
sa.Column("keep", sa.JSON, nullable=False, server_default="[]"),
sa.Column("drop", sa.JSON, nullable=False, server_default="[]"),
sa.Column("series_cap", sa.Integer, nullable=False, server_default="1000"),
sa.Column("enabled", sa.Boolean, nullable=False, server_default=sa.true()),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("last_scraped_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_ok_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text, nullable=True),
sa.Column("last_kept", sa.Integer, nullable=False, server_default="0"),
sa.Column("last_dropped", sa.Integer, nullable=False, server_default="0"),
sa.Column("last_capped", sa.Integer, nullable=False, server_default="0"),
)
def downgrade() -> None:
op.drop_table("prometheus_targets")
plugins/prometheus/migrations/versions/prometheus_002_templates.py
-33
View File
@@ -1,33 +0,0 @@
"""Widget templates an admin writes or copies in are rows
Each row is one template's YAML and its name, beside the bundled templates
that ship as files (#5692).
Revision ID: prometheus_002_templates
Revises: prometheus_001_targets
Create Date: 2026-10-14
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "prometheus_002_templates"
down_revision: Union[str, None] = "prometheus_001_targets"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"prometheus_widget_templates",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("name", sa.String(80), nullable=False, unique=True),
sa.Column("text", sa.Text, nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
op.drop_table("prometheus_widget_templates")
plugins/prometheus/models.py
-59
View File
@@ -1,59 +0,0 @@
# plugins/prometheus/models.py
from __future__ import annotations
from datetime import datetime, timezone
from sqlalchemy import JSON, Boolean, DateTime, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from steward.core.ids import time_ordered_id
from steward.models.base import Base
class PrometheusTarget(Base):
"""A /metrics endpoint Steward scrapes, edited on the Prometheus page.
The bearer token is stored encrypted. keep and drop are lists of name
patterns ("fabledarchives_*") an admin chooses: Steward does the
filtering, the endpoint serves everything. The last_* columns are the
newest scrape's outcome, written by the scraper, so a page can say why a
target isn't answering without asking it.
"""
__tablename__ = "prometheus_targets"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
url: Mapped[str] = mapped_column(String(500), nullable=False)
token: Mapped[str] = mapped_column(Text, nullable=False, default="")
verify_tls: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
interval_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=60)
keep: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
drop: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
series_cap: Mapped[int] = mapped_column(Integer, nullable=False, default=1000)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
last_scraped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_ok_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
last_kept: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
last_dropped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
last_capped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
class PrometheusWidgetTemplate(Base):
"""A widget template an admin wrote or copied in (#5692), beside the
bundled ones in widgets/*.yaml. `text` is the YAML as written; `name` is
its `name:`, kept here so the library lists and orders without reading
every template. Only a template that passes the checker is saved. Its key
in a widget's settings is "custom:<id>", so renaming it keeps its widgets.
"""
__tablename__ = "prometheus_widget_templates"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
name: Mapped[str] = mapped_column(String(80), nullable=False, unique=True)
text: Mapped[str] = mapped_column(Text, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
plugins/prometheus/plugin.yaml
-17
View File
@@ -1,17 +0,0 @@
# plugins/prometheus/plugin.yaml
name: prometheus
version: "1.0.0"
description: "Scrapes Prometheus /metrics endpoints into Steward's metrics, so any service that exposes them can be charted and alerted on."
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/FabledSteward"
homepage: "https://git.fabledsword.com/bvandeusen/FabledSteward/src/branch/dev/plugins/prometheus"
tags:
- prometheus
- metrics
- monitoring
# Targets are added, tested and edited on the Prometheus page
# (/plugins/prometheus/); each carries its own interval.
config: {}
plugins/prometheus/routes.py
-582
View File
@@ -1,582 +0,0 @@
"""The Prometheus pages. Every GET reads Steward's own records: the target
rows the editor writes and the scraper annotates, and the series it stores.
Only Test, an admin's explicit action, asks an endpoint anything (System
178's ruling)."""
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from quart import Blueprint, Response, abort, current_app, redirect, render_template, request, session
from sqlalchemy import delete, func, select, update
from steward.auth.middleware import require_role
from steward.core.audit import log_audit
from steward.core.crypto import decrypt_secret
from steward.core.metric_series import latest_values, points
from steward.models.metrics import PluginMetric, PluginMetricLatest
from steward.models.users import UserRole
from . import library
from .models import PrometheusTarget, PrometheusWidgetTemplate
from .scraper import ScrapeError, scrape
from .service import (OWN_METRICS, SOURCE, FormError, fold_states, form_values, map_families,
parse_form, patterns, shown_value, target_state)
prometheus_bp = Blueprint("prometheus", __name__, template_folder="templates")
_REACH_ORDER = {"down": 0, "stale": 1, "pending": 2, "up": 3, "off": 4}
def _actor() -> tuple[str | None, str]:
return session.get("user_id"), session.get("username", "")
async def _all_targets(db) -> list[PrometheusTarget]:
return list((await db.execute(select(PrometheusTarget).order_by(PrometheusTarget.name))).scalars().all())
def view(target: PrometheusTarget, own: dict, now: datetime) -> dict:
"""What a page says about a target: its state in words and the numbers
from its newest scrape."""
state = target_state(target, now)
return {"id": target.id, "name": target.name, "url": target.url, "reach": state.reach,
"says": state.says, "scraped_at": target.last_scraped_at, "ok_at": target.last_ok_at,
"interval": target.interval_seconds, "kept": target.last_kept,
"dropped": target.last_dropped, "capped": target.last_capped, "cap": target.series_cap,
"keep": list(target.keep or []), "drop": list(target.drop or []),
"scrape_ms": own.get("scrape_ms")}
@prometheus_bp.get("/")
@require_role(UserRole.viewer)
async def index():
async with current_app.db_sessionmaker() as db:
targets = await _all_targets(db)
own = {}
if targets:
rows = (await db.execute(select(PluginMetricLatest).where(
PluginMetricLatest.source_module == SOURCE,
PluginMetricLatest.resource_name.in_([t.name for t in targets]),
PluginMetricLatest.metric_name == "scrape_ms"))).scalars()
own = {r.resource_name: {"scrape_ms": r.value} for r in rows}
now = datetime.now(timezone.utc)
shown = [view(t, own.get(t.name, {}), now) for t in targets]
shown.sort(key=lambda v: (_REACH_ORDER[v["reach"]], v["name"].lower()))
return await render_template("prometheus/index.html", targets=shown)
# ── The editor ────────────────────────────────────────────────────────────────
async def _render_form(target: PrometheusTarget | None, values: dict, error: str | None = None,
status: int = 200):
return await render_template("prometheus/form.html", target=target, v=values, error=error), status
def _typed(form, target: PrometheusTarget | None = None) -> dict:
"""What was typed, to show the form again with an error. The token is not
echoed back."""
return {"name": form.get("name", ""), "url": form.get("url", ""),
"interval_seconds": form.get("interval_seconds", ""), "series_cap": form.get("series_cap", ""),
"keep": form.get("keep", ""), "drop": form.get("drop", ""),
"verify_tls": bool(form.get("verify_tls")), "enabled": bool(form.get("enabled")),
"has_token": bool(target and target.token)}
async def _name_taken(db, name: str, other_than: str | None = None) -> bool:
q = select(PrometheusTarget.id).where(PrometheusTarget.name == name)
if other_than:
q = q.where(PrometheusTarget.id != other_than)
return (await db.execute(q)).first() is not None
async def _target_or_404(db, target_id: str) -> PrometheusTarget:
target = await db.get(PrometheusTarget, target_id)
if target is None:
abort(404)
return target
@prometheus_bp.get("/targets/new")
@require_role(UserRole.admin)
async def new_target():
return await _render_form(None, form_values(None))
@prometheus_bp.post("/targets/new")
@require_role(UserRole.admin)
async def create_target():
form = await request.form
try:
values = parse_form(form)
except FormError as exc:
return await _render_form(None, _typed(form), str(exc), 422)
async with current_app.db_sessionmaker() as db:
async with db.begin():
if await _name_taken(db, values["name"]):
return await _render_form(None, _typed(form),
f"A target is already named “{values['name']}”.", 422)
target = PrometheusTarget(**values)
db.add(target)
await db.flush()
target_id = target.id
await log_audit(current_app, *_actor(), "prometheus.target_created", entity_type="prometheus_target",
entity_id=target_id, detail={"name": values["name"], "url": values["url"]})
return redirect(f"/plugins/prometheus/targets/{target_id}")
@prometheus_bp.get("/targets/<target_id>/edit")
@require_role(UserRole.admin)
async def edit_target(target_id: str):
async with current_app.db_sessionmaker() as db:
target = await _target_or_404(db, target_id)
return await _render_form(target, form_values(target))
@prometheus_bp.post("/targets/<target_id>/edit")
@require_role(UserRole.admin)
async def save_target(target_id: str):
form = await request.form
async with current_app.db_sessionmaker() as db:
async with db.begin():
target = await _target_or_404(db, target_id)
try:
values = parse_form(form, target)
except FormError as exc:
return await _render_form(target, _typed(form, target), str(exc), 422)
if await _name_taken(db, values["name"], other_than=target.id):
return await _render_form(target, _typed(form, target),
f"A target is already named “{values['name']}”.", 422)
old_name = target.name
for column, value in values.items():
setattr(target, column, value)
# Where it is or what is kept may have changed: the last outcome no
# longer describes it, so the page says "not scraped yet" until the next scrape.
target.last_scraped_at = target.last_ok_at = target.last_error = None
target.last_kept = target.last_dropped = target.last_capped = 0
if old_name != target.name:
# Its history is stored under its name, and its labelled series
# under "<name>:…": move them with the rename.
for model in (PluginMetric, PluginMetricLatest):
await db.execute(update(model).where(
model.source_module == SOURCE, model.resource_name == old_name
).values(resource_name=target.name))
# "<old>:labels" keeps its ":labels" (substr is 1-based).
await db.execute(update(model).where(
model.source_module == SOURCE,
model.resource_name.startswith(old_name + ":", autoescape=True),
).values(resource_name=target.name + func.substr(model.resource_name, len(old_name) + 1)))
from . import scheduler
scheduler.RATES.forget(target_id)
detail = {"name": values["name"]}
if old_name != values["name"]:
detail["old_name"] = old_name
await log_audit(current_app, *_actor(), "prometheus.target_updated", entity_type="prometheus_target",
entity_id=target_id, detail=detail)
return redirect(f"/plugins/prometheus/targets/{target_id}")
@prometheus_bp.post("/targets/<target_id>/delete")
@require_role(UserRole.admin)
async def delete_target(target_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
target = await _target_or_404(db, target_id)
name = target.name
await db.delete(target)
# Its newest values go with it, so nothing reads it as a target
# that stopped answering. Its history ages out with retention.
await db.execute(delete(PluginMetricLatest).where(
PluginMetricLatest.source_module == SOURCE,
(PluginMetricLatest.resource_name == name)
| PluginMetricLatest.resource_name.startswith(name + ":", autoescape=True)))
from . import scheduler
scheduler.RATES.forget(target_id)
await log_audit(current_app, *_actor(), "prometheus.target_deleted", entity_type="prometheus_target",
entity_id=target_id, detail={"name": name})
return redirect("/plugins/prometheus/")
TEST_FAMILIES_SHOWN = 40
@prometheus_bp.post("/targets/test")
@require_role(UserRole.admin)
async def test_target():
"""Scrape the (unsaved) form's address once and say what would be stored.
An admin's explicit action, so it may contact the endpoint."""
form = await request.form
existing = None
if form.get("target_id"):
async with current_app.db_sessionmaker() as db:
existing = await db.get(PrometheusTarget, form["target_id"])
url = (form.get("url") or "").strip()
token = (form.get("token") or "").strip()
if not token and existing is not None and existing.token and not form.get("clear_token"):
token = decrypt_secret(existing.token, context=f"prometheus target {existing.name} token")
found: dict = {}
if not url.startswith(("http://", "https://")):
found["error"] = "Enter the address first: http://… or https://…"
else:
try:
families = await scrape(url, token, bool(form.get("verify_tls")))
except ScrapeError as exc:
found["error"] = str(exc)
else:
try:
cap = max(1, int(form.get("series_cap") or 1000))
except ValueError:
cap = 1000
keep, drop = patterns(form.get("keep") or ""), patterns(form.get("drop") or "")
mapped = map_families("t", families, keep, drop, cap)
kept_names = {}
for s in mapped.series:
kept_names[s.metric] = kept_names.get(s.metric, 0) + 1
found = {"families": len(families), "kept": len(mapped.series), "dropped": mapped.dropped,
"capped": mapped.capped,
"names": sorted(kept_names.items())[:TEST_FAMILIES_SHOWN],
"more": max(0, len(kept_names) - TEST_FAMILIES_SHOWN)}
return await render_template("prometheus/_test.html", found=found, url=url)
# ── The target page and one series ───────────────────────────────────────────
def group_series(target_name: str, latest: list) -> list[dict]:
"""The target's stored series grouped by metric, each with its rows:
the labels that tell them apart, and the newest value. A state set is
folded to one row per thing it describes, naming the current state
(`states` set on the group)."""
groups: dict[str, list] = {}
for row in latest:
if row.resource_name == target_name and row.metric_name in OWN_METRICS:
continue
labels = row.resource_name[len(target_name) + 1:] if row.resource_name != target_name else ""
groups.setdefault(row.metric_name, []).append({
"labels": labels, "resource": row.resource_name, "at": row.recorded_at,
"value": row.value, **shown_value(row.metric_name, row.value)})
out = []
for metric, rows in sorted(groups.items()):
folded = fold_states(metric, rows)
if folded is not None:
out.append({"metric": metric, "states": True, "rows": [
{"labels": f["labels"], "resource": f["resource"], "moment": None,
"text": f["state"] or "none", "states": f["states"]} for f in folded]})
else:
out.append({"metric": metric, "states": False,
"rows": sorted(rows, key=lambda r: r["labels"])})
return out
@prometheus_bp.get("/targets/<target_id>")
@require_role(UserRole.viewer)
async def target_detail(target_id: str):
q = (request.args.get("q") or "").strip()
async with current_app.db_sessionmaker() as db:
target = await _target_or_404(db, target_id)
latest = await latest_values(db, SOURCE, target.name, include_children=True)
own = {r.metric_name: r.value for r in latest if r.resource_name == target.name
and r.metric_name in OWN_METRICS}
groups = group_series(target.name, latest)
total = sum(len(g["rows"]) for g in groups)
if q:
groups = [g for g in groups if q.lower() in g["metric"].lower()]
now = datetime.now(timezone.utc)
return await render_template("prometheus/target.html", target=view(target, own, now), groups=groups,
total=total, q=q)
RANGES = (1, 6, 24, 168)
@prometheus_bp.get("/targets/<target_id>/series")
@require_role(UserRole.viewer)
async def series_detail(target_id: str):
resource = request.args.get("resource") or ""
metric = request.args.get("metric") or ""
hours = request.args.get("hours", type=int, default=24)
if hours not in RANGES:
hours = 24
async with current_app.db_sessionmaker() as db:
target = await _target_or_404(db, target_id)
if resource != target.name and not resource.startswith(target.name + ":"):
abort(404)
now_row = await db.get(PluginMetricLatest, (SOURCE, resource, metric))
series = await points(db, SOURCE, resource, metric,
datetime.now(timezone.utc) - timedelta(hours=hours))
labels = resource[len(target.name) + 1:] if resource != target.name else ""
stamp_axis = metric.endswith("_timestamp_seconds")
return await render_template(
"prometheus/series.html", target=target, resource=resource, metric=metric, labels=labels,
now=shown_value(metric, now_row.value if now_row else None), hours=hours, ranges=RANGES,
points_json=json.dumps([(t.isoformat(), v) for t, v in series]), stamp_axis=stamp_axis)
# ── The dashboard widget ─────────────────────────────────────────────────────
SPARK_HOURS = 24
def spark_heights(values: list[float]) -> list[float]:
"""Bar heights in % of the series' own peak over the window (System
154's ruling on small history bars); a flat zero draws flat."""
peak = max((abs(v) for v in values), default=0.0)
return [round(100.0 * abs(v) / peak, 1) if peak else 0.0 for v in values]
async def draw(db, template, target: PrometheusTarget, now: datetime):
"""`template` drawn over `target`'s stored values, with the last day's bars
for the items that ask for them. The dashboard widget and the template
editor's preview both draw through here."""
from .template_view import render
latest = await latest_values(db, SOURCE, target.name, include_children=True)
shown = render(template, target.name, latest, now)
if shown.wants:
since = now - timedelta(hours=SPARK_HOURS)
history = {}
for resource, metric in shown.wants:
got = await points(db, SOURCE, resource, metric, since, target=SPARK_HOURS)
history[(resource, metric)] = spark_heights([v for _, v in got])
shown = render(template, target.name, latest, now, history=history)
return shown
@prometheus_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""The Prometheus dashboard widget: one target drawn by one template
(System 181's ruling), in widget shape. Reads stored values only."""
target_id = request.args.get("target") or ""
key = request.args.get("template") or ""
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
target = await db.get(PrometheusTarget, target_id) if target_id else None
entry = await library.find(db, key) if key else None
problem = ("This widget has no target; remove it and add it again with one." if not target_id
else "The target this widget showed has been deleted." if target is None
else "This widget has no template; remove it and add it again with one." if not key
else "The template this widget used is gone." if entry is None
else f"The template “{entry.name}” has a problem: {entry.problems[0]}"
if entry.template is None else "")
if problem:
return await render_template("prometheus/widget.html", problem=problem, target=None,
view=None)
shown = await draw(db, entry.template, target, now)
return await render_template("prometheus/widget.html", problem="", view=shown,
target=view_target(target, now), spark_hours=SPARK_HOURS)
def view_target(target: PrometheusTarget, now: datetime) -> dict:
state = target_state(target, now)
return {"id": target.id, "name": target.name, "reach": state.reach, "says": state.says,
"ok_at": target.last_ok_at, "scraped_at": target.last_scraped_at}
# ── The template library (#5692) ─────────────────────────────────────────────
# Bundled templates are read-only: they can be looked at, exported and
# duplicated. An admin writes, pastes or uploads their own, edits and deletes
# them. Nothing is fetched from anywhere (System 181's ruling).
def _shown_entry(entry, used: int) -> dict:
return {"key": entry.key, "id": entry.id, "name": entry.name, "bundled": entry.bundled,
"version": entry.template.version if entry.template else None,
"description": entry.template.description if entry.template else "",
"problem": str(entry.problems[0]) if entry.problems else "",
"updated_at": entry.updated_at, "used": used}
@prometheus_bp.get("/templates/")
@require_role(UserRole.viewer)
async def templates_index():
async with current_app.db_sessionmaker() as db:
entries = await library.entries(db)
used = await library.used_by(db)
shown = [_shown_entry(e, used.get(e.key, 0)) for e in entries]
# One that no longer passes the checker first: its widgets say so too.
shown.sort(key=lambda t: (not t["problem"], t["name"].lower()))
return await render_template("prometheus/templates.html", templates=shown)
async def _editor(entry=None, text: str = "", problems=(), note: str = "", status: int = 200,
saving: bool = False):
"""The template page: an admin's editor, or a read-only view of a
bundled template (or of any template, to a viewer). Both preview it."""
async with current_app.db_sessionmaker() as db:
targets = await library.target_options(db)
used = (await library.used_by(db)).get(entry.key, 0) if entry is not None else 0
editable = session.get("user_role") == "admin" and (entry is None or not entry.bundled)
return await render_template(
"prometheus/template.html", entry=entry, title=entry.name if entry else "New template",
text=text if entry is None or text else entry.text,
problems=[str(p) for p in problems], note=note, targets=targets, editable=editable,
used=used, saving=saving), status
async def _entry_or_404(db, key: str):
entry = await library.find(db, key)
if entry is None:
abort(404)
return entry
@prometheus_bp.get("/templates/new")
@require_role(UserRole.admin)
async def new_template():
"""A blank editor, or a copy of another template (?from=<key>) under a
name nobody has."""
source = request.args.get("from") or ""
if not source:
return await _editor()
async with current_app.db_sessionmaker() as db:
entry = await _entry_or_404(db, source)
taken = {e.name for e in await library.entries(db)}
return await _editor(text=library.renamed(entry.text, library.copy_name(entry.name, taken)),
note=f"A copy of “{entry.name}”. Nothing is saved until you press Save.")
async def _checked(db, text: str, other_than: str | None = None):
"""The template `text` describes and its problems: the checker's, and a
name another template already has."""
from .template_format import Problem, TemplateError, check
try:
template = check(text)
except TemplateError as exc:
return None, exc.problems
if (await library.name_taken(db, template.name, other_than)
or any(e.name == template.name for e in library.bundled())):
line = next((i for i, ln in enumerate(text.splitlines(), 1) if ln.startswith("name:")), 0)
return None, [Problem(line, f"A template is already named “{template.name}”; give this one its own name.")]
return template, []
def _typed_text(form) -> str:
return (form.get("text") or "").replace("\r\n", "\n")
@prometheus_bp.post("/templates/new")
@require_role(UserRole.admin)
async def create_template():
form = await request.form
text = _typed_text(form)
async with current_app.db_sessionmaker() as db:
async with db.begin():
template, problems = await _checked(db, text)
if problems:
return await _editor(text=text, problems=problems, status=422, saving=True)
row = PrometheusWidgetTemplate(name=template.name, text=text)
db.add(row)
await db.flush()
row_id = row.id
await log_audit(current_app, *_actor(), "prometheus.template_created", entity_type="prometheus_template",
entity_id=row_id, detail={"name": template.name, "version": template.version})
return redirect(f"/plugins/prometheus/templates/custom:{row_id}")
UPLOAD_LIMIT = 64 * 1024
@prometheus_bp.post("/templates/upload")
@require_role(UserRole.admin)
async def upload_template():
"""A .yaml file read into a new editor. Nothing is saved until Save."""
files = await request.files
upload = files.get("file")
if upload is None or not upload.filename:
return await _editor(problems=["Choose a .yaml file to upload."], status=422)
raw = upload.read(UPLOAD_LIMIT + 1)
if len(raw) > UPLOAD_LIMIT:
return await _editor(problems=[f"A template is at most {UPLOAD_LIMIT // 1024} KB."], status=422)
try:
text = raw.decode("utf-8-sig").replace("\r\n", "\n")
except UnicodeDecodeError:
return await _editor(problems=[f"{upload.filename} isn't a text file (UTF-8)."], status=422)
return await _editor(text=text, note=f"Read from {upload.filename}. Nothing is saved until you press Save.")
@prometheus_bp.get("/templates/<key>")
@require_role(UserRole.viewer)
async def template_detail(key: str):
async with current_app.db_sessionmaker() as db:
entry = await _entry_or_404(db, key)
return await _editor(entry, problems=entry.problems)
async def _custom_or_404(db, template_id: str) -> PrometheusWidgetTemplate:
row = await db.get(PrometheusWidgetTemplate, template_id)
if row is None:
abort(404)
return row
@prometheus_bp.post("/templates/custom:<template_id>/edit")
@require_role(UserRole.admin)
async def save_template(template_id: str):
form = await request.form
text = _typed_text(form)
async with current_app.db_sessionmaker() as db:
async with db.begin():
row = await _custom_or_404(db, template_id)
template, problems = await _checked(db, text, other_than=row.id)
if problems:
entry = library.custom_entry(row)
return await _editor(entry, text=text, problems=problems, status=422, saving=True)
old_name = row.name
row.name, row.text = template.name, text
row.updated_at = datetime.now(timezone.utc)
detail = {"name": template.name, "version": template.version}
if old_name != template.name:
detail["old_name"] = old_name
await log_audit(current_app, *_actor(), "prometheus.template_updated", entity_type="prometheus_template",
entity_id=template_id, detail=detail)
return redirect(f"/plugins/prometheus/templates/custom:{template_id}")
@prometheus_bp.post("/templates/custom:<template_id>/delete")
@require_role(UserRole.admin)
async def delete_template(template_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
row = await _custom_or_404(db, template_id)
name = row.name
await db.delete(row)
await log_audit(current_app, *_actor(), "prometheus.template_deleted", entity_type="prometheus_template",
entity_id=template_id, detail={"name": name})
return redirect("/plugins/prometheus/templates/")
@prometheus_bp.get("/templates/<key>/export")
@require_role(UserRole.viewer)
async def export_template(key: str):
"""The template's YAML as a file, to share or keep."""
async with current_app.db_sessionmaker() as db:
entry = await _entry_or_404(db, key)
return Response(entry.text, mimetype="application/yaml", headers={
"Content-Disposition": f'attachment; filename="{library.file_name(entry.name)}"'})
@prometheus_bp.post("/templates/preview")
@require_role(UserRole.viewer)
async def preview_template():
"""The editor's live preview: what's typed, checked, and drawn over a
target's stored values as the widget would draw it. Saves nothing."""
from .template_format import TemplateError, check
form = await request.form
text = _typed_text(form)
now = datetime.now(timezone.utc)
try:
template = check(text) if text.strip() else None
except TemplateError as exc:
return await render_template("prometheus/_preview.html", problems=[str(p) for p in exc.problems],
view=None, target=None)
if template is None:
return await render_template("prometheus/_preview.html", problems=[], view=None, target=None,
empty=True)
async with current_app.db_sessionmaker() as db:
target = await db.get(PrometheusTarget, form.get("target") or "") if form.get("target") else None
if target is None:
return await render_template("prometheus/_preview.html", problems=[], view=None, target=None,
template=template)
shown = await draw(db, template, target, now)
return await render_template("prometheus/_preview.html", problems=[], view=shown, template=template,
target=view_target(target, now), spark_hours=SPARK_HOURS)
plugins/prometheus/scheduler.py
-163
View File
@@ -1,163 +0,0 @@
# plugins/prometheus/scheduler.py
from __future__ import annotations
import asyncio
import logging
import time
from datetime import datetime, timezone
from typing import TYPE_CHECKING
from sqlalchemy import select
from steward.core.counter_rates import CounterRates
from steward.core.scheduler import ScheduledTask
if TYPE_CHECKING:
from quart import Quart
logger = logging.getLogger(__name__)
# How often the scheduler looks for due targets. A target is scraped within
# one tick of its interval, so this is the cadence's worst-case lateness.
TICK_SECONDS = 5
# A scrape past this is given up, whatever the client's own timeouts did:
# a slow drip of bytes can keep each read inside its timeout for ever.
SCRAPE_DEADLINE_SECONDS = 30
# Counters' previous samples, for their rates.
RATES = CounterRates()
def make_scrape_task(app: "Quart") -> ScheduledTask:
async def tick() -> None:
await scrape_due(app)
return ScheduledTask(name="prometheus_scrape", coro_factory=tick,
interval_seconds=TICK_SECONDS, run_on_startup=True)
def is_due(target, now: datetime) -> bool:
"""Whether a target's interval has elapsed since its last scrape, with
half a tick of slack so tick jitter doesn't push it back a whole tick."""
if not target.enabled:
return False
if target.last_scraped_at is None:
return True
elapsed = (now - target.last_scraped_at).total_seconds()
return elapsed >= target.interval_seconds - TICK_SECONDS / 2
async def scrape_due(app: "Quart", scrape=None) -> None:
"""Scrape every due target at once, then write each one's outcome to its
row and its series to the metric tables. `scrape` stands in for
scraper.scrape in tests."""
from steward.core.crypto import decrypt_secret
from .models import PrometheusTarget
from .scraper import ScrapeError
from .scraper import scrape as fetch
scrape = scrape or fetch
now = datetime.now(timezone.utc)
async with app.db_sessionmaker() as session:
targets = [t for t in (await session.execute(select(PrometheusTarget))).scalars().all()
if is_due(t, now)]
if not targets:
return
async def one(t):
started = time.monotonic()
token = decrypt_secret(t.token, context=f"prometheus target {t.name} token")
try:
families = await asyncio.wait_for(scrape(t.url, token, t.verify_tls),
SCRAPE_DEADLINE_SECONDS)
return families, (time.monotonic() - started) * 1000
except TimeoutError:
return ScrapeError(f"Gave up after {SCRAPE_DEADLINE_SECONDS} s."), \
(time.monotonic() - started) * 1000
except Exception as exc: # noqa: BLE001 — every failure is the target's outcome
return exc, (time.monotonic() - started) * 1000
results = await asyncio.gather(*(one(t) for t in targets))
await record(app, targets, results, datetime.now(timezone.utc))
async def forget_gone(session, target: str, kept: set[tuple[str, str]]) -> None:
"""Drop the newest value of each series of `target` that its last scrape
didn't keep: the endpoint stopped reporting it (an old build_info after an
upgrade, a deleted job), or a filter or the cap now leaves it out. Pages,
widgets and query_metrics read the newest values, so they show what the
endpoint says now. History stays for charts and ages out with retention.
Alert rules keep their own last-sample time, so one on a gone series
still goes no_data."""
from sqlalchemy import delete, tuple_
from steward.models.metrics import PluginMetricLatest
from .service import SOURCE
L = PluginMetricLatest
stored = (await session.execute(select(L.resource_name, L.metric_name).where(
L.source_module == SOURCE,
(L.resource_name == target) | L.resource_name.startswith(target + ":", autoescape=True)))).all()
gone = [(r, m) for r, m in stored if (r, m) not in kept]
if gone:
await session.execute(delete(L).where(
L.source_module == SOURCE, tuple_(L.resource_name, L.metric_name).in_(gone)))
async def record(app: "Quart", targets, results, now: datetime) -> None:
"""Each target's outcome on its row, and its kept series (counters as
rates) plus its own is_up / scrape_ms / series counts as metrics. After a
good scrape, the series it no longer has are forgotten (forget_gone); a
failed scrape says nothing about them, so they stay."""
from steward.core.alerts import MetricBatch
from .models import PrometheusTarget
from .scraper import ScrapeError
from .service import (IS_UP, OWN_METRICS, SCRAPE_MS, SERIES_CAPPED, SERIES_KEPT, SOURCE,
map_families)
async with app.db_sessionmaker() as session:
async with session.begin():
batch = MetricBatch()
answered: list[tuple[str, set[tuple[str, str]]]] = []
for target, (result, ms) in zip(targets, results):
row = await session.get(PrometheusTarget, target.id)
if row is None: # deleted while it was being scraped
continue
row.last_scraped_at = now
if isinstance(result, BaseException):
if not isinstance(result, ScrapeError):
logger.warning("Prometheus scrape of %s failed: %r", target.name, result)
row.last_error = str(result) or "The scrape failed."
batch.add(source_module=SOURCE, resource_name=row.name, metric_name=IS_UP,
value=0.0, recorded_at=now)
continue
mapped = map_families(row.name, result, list(row.keep or []), list(row.drop or []),
row.series_cap)
row.last_error = None
row.last_ok_at = now
row.last_kept = len(mapped.series)
row.last_dropped = mapped.dropped
row.last_capped = mapped.capped
# A counter's first scrape gives no rate yet; it is still kept.
answered.append((row.name, {(x.resource, x.metric) for x in mapped.series}
| {(row.name, m) for m in OWN_METRICS}))
for series in mapped.series:
value = series.value
if series.counter:
value = RATES.rate((row.id, series.resource, series.metric), value, now)
if value is None:
continue
batch.add(source_module=SOURCE, resource_name=series.resource,
metric_name=series.metric, value=value, recorded_at=now)
for metric_name, value in [(IS_UP, 1.0), (SCRAPE_MS, ms),
(SERIES_KEPT, len(mapped.series)),
(SERIES_CAPPED, mapped.capped)]:
batch.add(source_module=SOURCE, resource_name=row.name,
metric_name=metric_name, value=value, recorded_at=now)
await batch.record(session)
for name, kept in answered:
await forget_gone(session, name, kept)
plugins/prometheus/scraper.py
-58
View File
@@ -1,58 +0,0 @@
# plugins/prometheus/scraper.py
"""Fetching one /metrics endpoint. Every failure comes back as one sentence
a page can show, never a stack trace."""
from __future__ import annotations
import httpx
from steward.core import prometheus
TIMEOUT_SECONDS = 10.0
# A /metrics page past this is not one to store a series of; reading stops.
MAX_BYTES = 20 * 1024 * 1024
ACCEPT = "text/plain;version=0.0.4;q=1,*/*;q=0.1"
class ScrapeError(Exception):
"""Why a scrape got nothing, said to the person."""
async def scrape(url: str, token: str = "", verify_tls: bool = True,
transport: httpx.AsyncBaseTransport | None = None) -> dict[str, prometheus.Family]:
"""The endpoint's families. `transport` stands in for the network in tests."""
headers = {"Accept": ACCEPT}
if token:
headers["Authorization"] = f"Bearer {token}"
try:
async with httpx.AsyncClient(timeout=TIMEOUT_SECONDS, verify=verify_tls,
transport=transport, follow_redirects=False) as client:
async with client.stream("GET", url, headers=headers) as response:
if response.status_code in (401, 403):
raise ScrapeError(f"Answered {response.status_code}: the token was refused. "
"Check the token, or make a new one on the service.")
if response.status_code != 200:
raise ScrapeError(f"Answered {response.status_code} {response.reason_phrase}, not metrics.")
body = bytearray()
async for chunk in response.aiter_bytes():
body.extend(chunk)
if len(body) > MAX_BYTES:
raise ScrapeError(f"The page is over {MAX_BYTES // (1024 * 1024)} MB; "
"is this a /metrics address?")
except ScrapeError:
raise
except httpx.TimeoutException:
raise ScrapeError(f"No answer within {TIMEOUT_SECONDS:.0f} s.") from None
except httpx.ConnectError as exc:
raise ScrapeError(f"Couldn't connect: {_reason(exc)}.") from None
except httpx.HTTPError as exc:
raise ScrapeError(f"The request failed: {_reason(exc)}.") from None
families = prometheus.parse(body.decode("utf-8", errors="replace"))
if not families:
raise ScrapeError("It answered, but with no metrics in Prometheus format.")
return families
def _reason(exc: Exception) -> str:
text = str(exc).strip() or type(exc).__name__
return text.rstrip(".")
plugins/prometheus/service.py
-269
View File
@@ -1,269 +0,0 @@
"""Prometheus targets: the form a person edits, the row it becomes, how a
scrape's series become Steward metrics, and what a page says about the
newest scrape."""
from __future__ import annotations
import math
from dataclasses import dataclass, field
from datetime import datetime, timezone
from fnmatch import fnmatchcase
from steward.core import prometheus
from steward.core.crypto import encrypt_secret
from steward.core.formatting import metric_value
from .models import PrometheusTarget
SOURCE = "prometheus"
# What each target records about its own scrape, under its own name.
IS_UP = "is_up"
SCRAPE_MS = "scrape_ms"
SERIES_KEPT = "series_kept"
SERIES_CAPPED = "series_capped"
OWN_METRICS = (IS_UP, SCRAPE_MS, SERIES_KEPT, SERIES_CAPPED)
# Column widths of plugin_metrics; a longer name can't be stored.
RESOURCE_MAX = 255
METRIC_MAX = 128
INTERVAL_MIN, INTERVAL_MAX = 10, 3600
CAP_MIN, CAP_MAX = 1, 10000
class FormError(ValueError):
"""What is wrong with a submitted target form, said to the person."""
# ── Series → metrics ─────────────────────────────────────────────────────────
def wanted(name: str, keep: list[str], drop: list[str]) -> bool:
"""Whether a family is stored: it matches a keep pattern (or there are
none) and no drop pattern. Drop wins over keep."""
if keep and not any(fnmatchcase(name, p) for p in keep):
return False
return not any(fnmatchcase(name, p) for p in drop)
def resource_for(target: str, labels: tuple[tuple[str, str], ...]) -> str:
"""The resource a series is stored under: the target's name, then its
labels as key=value, so "Archives:target=Steward,state=late". A series
without labels is the target itself."""
if not labels:
return target
return target + ":" + ",".join(f"{k}={v}" for k, v in labels)
def rate_name(sample_name: str) -> str:
"""A counter is stored as a per-second rate: requests_total →
requests_per_sec, job_seconds_sum → job_seconds_sum_per_sec."""
base = sample_name[: -len("_total")] if sample_name.endswith("_total") else sample_name
return base + "_per_sec"
def label_pairs(labels: str) -> list[tuple[str, str]]:
"""resource_for's labels read back: "target=S,state=late" →
[("target", "S"), ("state", "late")]. A piece without "=" was part of the
value before it, a value that held a comma."""
pairs: list[tuple[str, str]] = []
for piece in labels.split(",") if labels else []:
key, eq, value = piece.partition("=")
if eq and key:
pairs.append((key, value))
elif pairs:
pairs[-1] = (pairs[-1][0], pairs[-1][1] + "," + piece)
return pairs
def state_label(metric: str, pairs: list[tuple[str, str]]) -> str | None:
"""Which label names a state set's state: `state`, or the metric's own
name as OpenMetrics writes a stateset (foo{foo="a"} 1)."""
keys = {k for k, _ in pairs}
for key in ("state", metric):
if key in keys:
return key
return None
def fold_states(metric: str, rows: list[dict]) -> list[dict] | None:
"""A state set's series folded to one row per thing it describes.
A state set is one series per possible state, the current one 1 and the
rest 0: Archives' fabledarchives_backup_target_state{target, state}. Each
row in `rows` has `labels` (resource_for's form), `value` and `resource`.
A metric is folded when its name ends in _state (or its state label is
its own name) and every series carries the state label and reads 0 or 1;
anything else is None, and shown series by series. The name matters: a
count split by a `state` label (fabledarchives_jobs{state="waiting"})
often reads 0 or 1 too, and is not a state set.
Each folded row: `labels` without the state, `state` (the one set to 1;
several are joined, none is ""), `states` (every state, in order) and
`resource` (the current state's series, else the first)."""
if not rows:
return None
sets: dict[str, list[tuple[str, dict]]] = {}
for row in rows:
pairs = label_pairs(row["labels"])
key = state_label(metric, pairs)
if key is None or row["value"] not in (0, 1):
return None
if key == "state" and not metric.endswith("_state"):
return None
rest = ",".join(f"{k}={v}" for k, v in pairs if k != key)
sets.setdefault(rest, []).append((dict(pairs)[key], row))
folded = []
for rest, members in sorted(sets.items()):
members.sort(key=lambda m: m[0])
current = [(s, r) for s, r in members if r["value"] == 1]
folded.append({"labels": rest, "state": ", ".join(s for s, _ in current),
"states": [s for s, _ in members],
"resource": (current or members)[0][1]["resource"]})
return folded
def shown_value(metric: str, value: float | None) -> dict:
"""A series' value in its unit, from the Prometheus naming conventions.
A *_timestamp_seconds is a moment: it comes back as `moment` for the page
to show as "3m ago" in the reader's zone; 0 there means "never"."""
if value is None:
return {"text": "—", "moment": None}
if metric.endswith("_timestamp_seconds"):
if value <= 0:
return {"text": "never", "moment": None}
return {"text": "", "moment": datetime.fromtimestamp(value, timezone.utc)}
return {"text": metric_value(metric, value), "moment": None}
@dataclass
class Series:
resource: str
metric: str
value: float
counter: bool
@dataclass
class Mapped:
series: list[Series] = field(default_factory=list)
dropped: int = 0 # left out by the keep/drop patterns
capped: int = 0 # past the series cap, or a name too long to store
def map_families(target: str, families: dict[str, prometheus.Family], keep: list[str],
drop: list[str], cap: int) -> Mapped:
"""The series of one scrape that Steward stores, in the endpoint's order.
Gauges and untyped samples are stored as they are; counters (and a
histogram's or summary's _sum and _count) are marked to become rates.
Histogram buckets and summary quantiles are skipped: Steward stores one
number per series, not a distribution. NaN and infinite values are
skipped as unmeasured."""
out = Mapped()
for family in families.values():
if not wanted(family.name, keep, drop):
out.dropped += len(family.samples)
continue
for sample in family.samples:
counter = family.type == "counter"
if family.type in ("histogram", "summary"):
if not sample.name.endswith(("_sum", "_count")):
continue
counter = True
if math.isnan(sample.value) or math.isinf(sample.value):
continue
metric = rate_name(sample.name) if counter else sample.name
resource = resource_for(target, sample.labels)
if len(out.series) >= cap or len(metric) > METRIC_MAX or len(resource) > RESOURCE_MAX:
out.capped += 1
continue
out.series.append(Series(resource, metric, sample.value, counter))
return out
# ── The form ─────────────────────────────────────────────────────────────────
def patterns(text: str) -> list[str]:
"""Name patterns from a textarea: one per line or comma-separated."""
out: list[str] = []
for line in (text or "").replace(",", "\n").splitlines():
p = line.strip()
if p and p not in out:
out.append(p)
return out
def form_values(target: PrometheusTarget | None) -> dict:
"""The form's fields for an existing target. The token is never sent back
to the browser; a blank token field on save keeps the stored one."""
if target is None:
return {"interval_seconds": 60, "series_cap": 1000, "verify_tls": True, "enabled": True,
"keep": "", "drop": ""}
return {"name": target.name, "url": target.url, "interval_seconds": target.interval_seconds,
"series_cap": target.series_cap, "verify_tls": target.verify_tls,
"enabled": target.enabled, "keep": "\n".join(target.keep or []),
"drop": "\n".join(target.drop or []), "has_token": bool(target.token)}
def _bounded(form, key: str, default: int, low: int, high: int, says: str) -> int:
try:
value = int(form.get(key) or default)
except ValueError:
raise FormError(says) from None
if not low <= value <= high:
raise FormError(says)
return value
def parse_form(form, existing: PrometheusTarget | None = None) -> dict:
"""The submitted form as the columns to store, the token encrypted.
Raises FormError saying what to fix."""
name = (form.get("name") or "").strip()
url = (form.get("url") or "").strip()
if not name:
raise FormError("Give the target a name.")
if len(name) > 100 or any(c in name for c in "/:,"):
raise FormError("A name is up to 100 characters, without / : or a comma.")
if not url.startswith(("http://", "https://")) or len(url) > 500:
raise FormError("The address starts with http:// or https://, e.g. http://archives:8081/metrics.")
interval = _bounded(form, "interval_seconds", 60, INTERVAL_MIN, INTERVAL_MAX,
f"Scrape every {INTERVAL_MIN} to {INTERVAL_MAX} seconds.")
cap = _bounded(form, "series_cap", 1000, CAP_MIN, CAP_MAX,
f"The series cap is between {CAP_MIN} and {CAP_MAX}.")
remove_token = bool(form.get("clear_token"))
typed = (form.get("token") or "").strip()
if typed:
token = encrypt_secret(typed)
elif existing is not None and not remove_token:
token = existing.token
else:
token = ""
return {"name": name, "url": url, "token": token,
"verify_tls": bool(form.get("verify_tls")), "interval_seconds": interval,
"keep": patterns(form.get("keep") or ""), "drop": patterns(form.get("drop") or ""),
"series_cap": cap, "enabled": bool(form.get("enabled"))}
# ── What the newest scrape says ──────────────────────────────────────────────
# A target not scraped for this many intervals has stopped being scraped
# (plugin turned off, scheduler stuck), whatever its last outcome said.
STALE_SCRAPES = 3
@dataclass
class TargetState:
reach: str # up | down | stale | pending | off
says: str # one line, in words
def target_state(target: PrometheusTarget, now: datetime) -> TargetState:
if not target.enabled:
return TargetState("off", "Not scraped: turned off.")
if target.last_scraped_at is None:
return TargetState("pending", "Not scraped yet.")
if (now - target.last_scraped_at).total_seconds() > STALE_SCRAPES * max(target.interval_seconds, 1):
return TargetState("stale", "Not scraped recently: the Prometheus plugin may be turned off.")
if target.last_error:
return TargetState("down", target.last_error)
return TargetState("up", "Answering.")
plugins/prometheus/template_format.py
-458
View File
@@ -1,458 +0,0 @@
"""Prometheus widget templates: the YAML a person writes, read and checked.
A template says which of a target's series a dashboard widget shows and
how. It is data only: it names series and lays them out in blocks, and
Steward draws them (template_view.py). It can't add a visual, a colour or a
formula; a new block is a Steward release (System 181's ruling). The
format, for people writing one: docs/plugins/prometheus-widgets.md.
name: Fabled Archives
version: 1
expects: fabledarchives_
blocks:
- states:
title: Backup targets
metric: fabledarchives_backup_target_state
name: target
healthy: [fresh]
quiet: [disabled, waiting]
warning: [late]
- values:
items:
- {label: Disk free, metric: fabledarchives_data_disk_free_bytes,
of: fabledarchives_data_disk_size_bytes, warn_below: 10%}
check(text) returns the Template, or every problem found, each with its
line and a sentence saying what to change.
"""
from __future__ import annotations
import re
from dataclasses import dataclass, field
import yaml
FORMAT = 1
MAX_BYTES = 64 * 1024
MAX_BLOCKS = 20
MAX_ITEMS = 30
_METRIC = re.compile(r"^[a-zA-Z_:][a-zA-Z0-9_:]*$")
_LABEL = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
_DURATION = re.compile(r"^(\d+(?:\.\d+)?)\s*([smhd]?)$")
_SECONDS = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400}
# What a unit shows a value as; the name is the suffix metric_value reads.
UNITS = {"bytes": "_bytes", "seconds": "_secs", "percent": "_pct", "per_sec": "_per_sec",
"count": "", "timestamp": "_timestamp_seconds"}
@dataclass
class Problem:
line: int
text: str
def __str__(self) -> str:
return f"Line {self.line}: {self.text}" if self.line else self.text
@dataclass
class Block:
kind: str # states | values | times | table | info
title: str
line: int
spec: dict = field(default_factory=dict)
@dataclass
class Template:
name: str
version: int
description: str = ""
expects: str = ""
blocks: list[Block] = field(default_factory=list)
text: str = ""
class TemplateError(ValueError):
def __init__(self, problems: list[Problem]):
self.problems = problems
super().__init__("; ".join(str(p) for p in problems))
# ── YAML with line numbers ───────────────────────────────────────────────────
class _Map(dict):
line = 0
lines: dict
class _List(list):
line = 0
lines: list
class _Loader(yaml.SafeLoader):
pass
def _construct_map(loader, node):
loader.flatten_mapping(node)
out = _Map()
out.line, out.lines = node.start_mark.line + 1, {}
for key_node, value_node in node.value:
key = loader.construct_object(key_node, deep=True)
if not isinstance(key, str):
raise yaml.constructor.ConstructorError(
None, None, "a key here should be a word, like `metric`", key_node.start_mark)
if key in out:
raise yaml.constructor.ConstructorError(
None, None, f"`{key}` is given twice", key_node.start_mark)
out[key] = loader.construct_object(value_node, deep=True)
out.lines[key] = key_node.start_mark.line + 1
return out
def _construct_list(loader, node):
out = _List(loader.construct_object(n, deep=True) for n in node.value)
out.line, out.lines = node.start_mark.line + 1, [n.start_mark.line + 1 for n in node.value]
return out
_Loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_map)
_Loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, _construct_list)
def _load(text: str) -> tuple[object, list[Problem]]:
if len(text.encode()) > MAX_BYTES:
return None, [Problem(0, f"A template is at most {MAX_BYTES // 1024} KB.")]
try:
# An alias can make a small file expand without end; a template has no use for one.
for event in yaml.parse(text, Loader=yaml.SafeLoader):
if isinstance(event, yaml.AliasEvent):
return None, [Problem(event.start_mark.line + 1,
"Anchors and aliases (&name, *name) aren't allowed; write the value out.")]
return yaml.load(text, Loader=_Loader), []
except yaml.constructor.ConstructorError as exc: # ours: a key twice, or not a word
mark = exc.problem_mark
return None, [Problem(mark.line + 1 if mark else 0, exc.problem[:1].upper() + exc.problem[1:] + ".")]
except yaml.MarkedYAMLError as exc:
mark = exc.problem_mark or exc.context_mark
return None, [Problem(mark.line + 1 if mark else 0, f"This isn't valid YAML: {exc.problem}.")]
except yaml.YAMLError as exc:
return None, [Problem(0, f"This isn't valid YAML: {exc}.")]
# ── Values ───────────────────────────────────────────────────────────────────
def duration(value) -> float | None:
"""Seconds from 90, "90s", "15m", "26h" or "2d"; None if it isn't one."""
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
return float(value) if value >= 0 else None
m = _DURATION.match(str(value).strip().lower())
return float(m.group(1)) * _SECONDS[m.group(2)] if m else None
@dataclass
class Limit:
"""A threshold: a number, or a share of the item's `of` ("10%")."""
value: float
share: bool = False
def limit(value) -> Limit | None:
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
return Limit(float(value))
text = str(value).strip()
if text.endswith("%"):
try:
return Limit(float(text[:-1]), share=True)
except ValueError:
return None
return None
# ── The checker ──────────────────────────────────────────────────────────────
class _Check:
def __init__(self):
self.problems: list[Problem] = []
def add(self, line: int, text: str) -> None:
self.problems.append(Problem(line, text))
def keys(self, m: _Map, allowed: tuple, required: tuple, where: str) -> bool:
ok = True
for key in m:
if key not in allowed:
self.add(m.lines[key], f"{where} has no `{key}`. It takes: {', '.join(allowed)}.")
ok = False
for key in required:
if key not in m:
self.add(m.line, f"{where} needs `{key}`.")
ok = False
return ok
def text(self, m: _Map, key: str, where: str, default: str = "") -> str:
if key not in m:
return default
value = m[key]
if isinstance(value, (dict, list)) or value is None:
self.add(m.lines[key], f"`{key}` in {where} should be text.")
return default
return str(value)
def metric(self, m: _Map, key: str, where: str) -> str:
value = m.get(key)
if not isinstance(value, str) or not _METRIC.match(value):
self.add(m.lines.get(key, m.line),
f"`{key}` in {where} should be a metric name as Steward stores it, like "
"fabledarchives_files (a counter is stored as <name>_per_sec).")
return ""
return value
def label(self, m: _Map, key: str, where: str) -> str:
if key not in m:
return ""
value = m[key]
if not isinstance(value, str) or not _LABEL.match(value):
self.add(m.lines[key], f"`{key}` in {where} should be a label name, like target.")
return ""
return value
def words(self, m: _Map, key: str, where: str) -> list[str]:
if key not in m:
return []
value = m[key]
if not isinstance(value, list) or not all(isinstance(v, (str, int)) for v in value):
self.add(m.lines[key], f"`{key}` in {where} should be a list, like [fresh, waiting].")
return []
if any(isinstance(v, bool) for v in value):
self.add(m.lines[key], f"`{key}` in {where} has a word YAML reads as true or false "
"(on, off, yes, no); put it in quotes, like 'off'.")
return []
return [str(v) for v in value]
def where(self, m: _Map, ctx: str) -> dict[str, str]:
if "where" not in m:
return {}
value = m["where"]
if not isinstance(value, dict) or not all(
isinstance(k, str) and _LABEL.match(k) and isinstance(v, (str, int, float))
and not isinstance(v, bool) for k, v in value.items()):
self.add(m.lines["where"], f"`where` in {ctx} should map labels to values, like "
"{kind: files}. Quote a value YAML would read as true or false.")
return {}
return {k: str(v) for k, v in value.items()}
def unit(self, m: _Map, ctx: str) -> str:
if "unit" not in m:
return ""
value = m["unit"]
if value not in UNITS:
self.add(m.lines["unit"], f"`unit` in {ctx} is one of: {', '.join(UNITS)}.")
return ""
return value
def span(self, m: _Map, key: str, ctx: str) -> float | None:
if key not in m:
return None
value = duration(m[key])
if value is None:
self.add(m.lines[key], f"`{key}` in {ctx} should be a span, like 26h, 90m or 2d.")
return value
def items(self, m: _Map, ctx: str) -> list[_Map]:
value = m.get("items")
if not isinstance(value, list) or not value:
self.add(m.lines.get("items", m.line), f"`items` in {ctx} should be a list of one or more items.")
return []
if len(value) > MAX_ITEMS:
self.add(m.lines["items"], f"{ctx} has {len(value)} items; at most {MAX_ITEMS}.")
out = []
for i, item in enumerate(value):
if not isinstance(item, dict):
self.add(value.lines[i], f"Each item in {ctx} should be a set of keys, like "
"{label: Disk free, metric: …}.")
else:
out.append(item)
return out
# Each block: the keys it takes, those it needs, and its title when none is given.
_BLOCKS = {
"states": (("title", "metric", "where", "name", "healthy", "quiet", "warning", "warn_past",
"crit_past", "columns"),
("metric", "healthy"), "States"),
"values": (("title", "items"), ("items",), "Figures"),
"times": (("title", "items"), ("items",), "Times"),
"table": (("title", "metric", "where", "rows", "columns", "unit"), ("metric", "rows", "columns"),
"Table"),
"info": (("title", "metric", "where", "labels"), ("metric",), "About"),
}
_VALUE_ITEM = ("label", "metric", "where", "name", "of", "unit", "warn_above", "crit_above",
"warn_below", "crit_below", "sparkline")
_TIME_ITEM = ("label", "metric", "where", "name", "warn_after", "crit_after")
_COLUMN = ("label", "metric", "unit")
def _states(c: _Check, m: _Map, ctx: str) -> dict:
spec = {"metric": c.metric(m, "metric", ctx), "where": c.where(m, ctx), "name": c.label(m, "name", ctx),
"healthy": c.words(m, "healthy", ctx), "quiet": c.words(m, "quiet", ctx),
"warning": c.words(m, "warning", ctx), "columns": [],
# A deadline per row (a *_timestamp_seconds): once it has passed, a
# healthy row reads as a warning or a problem. A job that says
# "running" for days is still overdue.
"warn_past": c.metric(m, "warn_past", ctx) if "warn_past" in m else "",
"crit_past": c.metric(m, "crit_past", ctx) if "crit_past" in m else ""}
if "healthy" in m and not spec["healthy"] and isinstance(m["healthy"], list):
c.add(m.lines["healthy"], f"`healthy` in {ctx} needs at least one state.")
seen: dict[str, str] = {}
for key in ("healthy", "quiet", "warning"):
for state in spec[key]:
if state in seen:
c.add(m.lines[key], f"“{state}” is in both `{seen[state]}` and `{key}` in {ctx}.")
seen[state] = key
if "columns" in m:
cols = m["columns"]
if not isinstance(cols, list):
c.add(m.lines["columns"], f"`columns` in {ctx} should be a list of {{label, metric}}.")
else:
for i, col in enumerate(cols):
where = f"a column of {ctx}"
if not isinstance(col, dict):
c.add(cols.lines[i], f"Each of `columns` in {ctx} should be {{label, metric}}.")
continue
if c.keys(col, _COLUMN, ("label", "metric"), where):
spec["columns"].append({"label": c.text(col, "label", where),
"metric": c.metric(col, "metric", where),
"unit": c.unit(col, where)})
return spec
def _value_item(c: _Check, item: _Map, ctx: str) -> dict | None:
where = f"an item of {ctx}"
if not c.keys(item, _VALUE_ITEM, ("label", "metric"), where):
return None
spec = {"label": c.text(item, "label", where), "metric": c.metric(item, "metric", where),
"where": c.where(item, where), "name": c.label(item, "name", where),
"of": c.metric(item, "of", where) if "of" in item else "", "unit": c.unit(item, where),
"sparkline": False}
if "sparkline" in item:
if not isinstance(item["sparkline"], bool):
c.add(item.lines["sparkline"], f"`sparkline` in {where} is true or false.")
spec["sparkline"] = item["sparkline"] is True
for key in ("warn_above", "crit_above", "warn_below", "crit_below"):
spec[key] = None
if key in item:
lim = limit(item[key])
if lim is None:
c.add(item.lines[key], f"`{key}` in {where} should be a number, or a share like 10%.")
elif lim.share and not spec["of"]:
c.add(item.lines[key], f"`{key}` in {where} is a share ({item[key]}), which needs `of`: "
"the metric it is a share of.")
spec[key] = lim
return spec
def _time_item(c: _Check, item: _Map, ctx: str) -> dict | None:
where = f"an item of {ctx}"
if not c.keys(item, _TIME_ITEM, ("label", "metric"), where):
return None
spec = {"label": c.text(item, "label", where), "metric": c.metric(item, "metric", where),
"where": c.where(item, where), "name": c.label(item, "name", where),
"warn_after": c.span(item, "warn_after", where), "crit_after": c.span(item, "crit_after", where)}
if spec["warn_after"] and spec["crit_after"] and spec["crit_after"] < spec["warn_after"]:
c.add(item.lines["crit_after"], f"`crit_after` in {where} should be later than `warn_after`.")
return spec
def _block(c: _Check, raw, line: int, n: int) -> Block | None:
if not isinstance(raw, dict) or len(raw) != 1:
c.add(line, f"Block {n} should be one kind with its settings under it, like "
f"`- states: {{metric: …}}`. The kinds are: {', '.join(_BLOCKS)}.")
return None
kind, body = next(iter(raw.items()))
if kind not in _BLOCKS:
c.add(line, f"Block {n} is “{kind}”; the kinds are: {', '.join(_BLOCKS)}.")
return None
if not isinstance(body, dict):
c.add(line, f"Block {n} ({kind}) should have its settings under it, like `metric: …`.")
return None
allowed, required, default_title = _BLOCKS[kind]
ctx = f"block {n} ({kind})"
if not c.keys(body, allowed, required, ctx.capitalize()):
return None
title = c.text(body, "title", ctx, default_title)
if kind == "states":
spec = _states(c, body, ctx)
elif kind in ("values", "times"):
reader = _value_item if kind == "values" else _time_item
spec = {"items": [s for s in (reader(c, i, ctx) for i in c.items(body, ctx)) if s]}
elif kind == "table":
spec = {"metric": c.metric(body, "metric", ctx), "where": c.where(body, ctx),
"rows": c.label(body, "rows", ctx), "columns": c.label(body, "columns", ctx),
"unit": c.unit(body, ctx)}
else:
spec = {"metric": c.metric(body, "metric", ctx), "where": c.where(body, ctx),
"labels": c.words(body, "labels", ctx)}
return Block(kind, title, line, spec)
_TOP = ("name", "description", "version", "format", "expects", "blocks")
def check(text: str) -> Template:
"""The template `text` describes. Raises TemplateError with every problem
found, each with its line, in the order they appear."""
raw, problems = _load(text)
if problems:
raise TemplateError(problems)
c = _Check()
if not isinstance(raw, dict):
raise TemplateError([Problem(1, "A template starts with its keys: name, version and blocks.")])
c.keys(raw, _TOP, ("name", "version", "blocks"), "A template")
name = c.text(raw, "name", "the template").strip()
if "name" in raw and not name:
c.add(raw.lines["name"], "Give the template a name.")
if len(name) > 80:
c.add(raw.lines["name"], "A name is at most 80 characters.")
version = raw.get("version")
if "version" in raw and (isinstance(version, bool) or not isinstance(version, int) or version < 1):
c.add(raw.lines["version"], "`version` is the template's own version: a whole number from 1. "
"Raise it when you change a template you have shared.")
if "format" in raw and raw["format"] != FORMAT:
c.add(raw.lines["format"], f"This Steward reads template format {FORMAT}.")
expects = c.text(raw, "expects", "the template")
if expects and not re.match(r"^[a-zA-Z_:][a-zA-Z0-9_:]*$", expects):
c.add(raw.lines["expects"], "`expects` should be the start of the metric names it reads, "
"like fabledarchives_.")
blocks: list[Block] = []
raw_blocks = raw.get("blocks")
if "blocks" in raw:
if not isinstance(raw_blocks, list) or not raw_blocks:
c.add(raw.lines["blocks"], "`blocks` should be a list of one or more blocks.")
else:
if len(raw_blocks) > MAX_BLOCKS:
c.add(raw.lines["blocks"], f"A template has at most {MAX_BLOCKS} blocks.")
for i, b in enumerate(raw_blocks):
block = _block(c, b, raw_blocks.lines[i], i + 1)
if block:
blocks.append(block)
if c.problems:
raise TemplateError(sorted(c.problems, key=lambda p: p.line))
return Template(name=name, version=version, description=c.text(raw, "description", "the template"),
expects=expects, blocks=blocks, text=text)
def problems(text: str) -> list[Problem]:
"""Every problem with `text`; empty when it is a template."""
try:
check(text)
except TemplateError as exc:
return exc.problems
return []
plugins/prometheus/template_view.py
-326
View File
@@ -1,326 +0,0 @@
"""A widget template drawn from a target's newest values: what the
Prometheus dashboard widget shows. Rows in, a view out; nothing is fetched
here, so the widget route reads the database and a test passes rows.
The view follows the widget shape (System 154): a verdict naming the worst
thing, then each block with its problems first and the healthy rest after.
Every row carries a tone: crit, warn, ok, or dim for a state the template
calls quiet.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime
from steward.core.formatting import human_duration
from .service import fold_states, label_pairs, shown_value
from .template_format import UNITS, Block, Limit, Template
TONE_ORDER = {"crit": 0, "warn": 1, "ok": 2, "dim": 3}
@dataclass
class Sample:
resource: str
metric: str
labels: str # as stored: "target=S,state=late"
pairs: dict[str, str]
value: float
at: datetime | None = None
@dataclass
class Row:
name: str
tone: str = "ok"
why: str = "" # what is wrong, in words: "is late", "below 10%"
text: str = "" # the value as shown, "—" when there is none
moment: datetime | None = None
bar: float | None = None # a share of `of`, 0 to 1
cells: list[dict] = field(default_factory=list)
resource: str = ""
metric: str = ""
history: list[float] | None = None
@dataclass
class BlockView:
kind: str
title: str
tone: str = "ok"
rows: list[Row] = field(default_factory=list)
columns: list[str] = field(default_factory=list)
note: str = "" # the block couldn't be drawn: no series, or not a state set
@dataclass
class View:
tone: str
verdict: str
blocks: list[BlockView]
wants: list[tuple[str, str]] = field(default_factory=list) # (resource, metric) for sparklines
note: str = ""
def samples(target: str, latest) -> dict[str, list[Sample]]:
"""A target's newest values by metric. `latest` is PluginMetricLatest
rows (resource_name, metric_name, value, recorded_at) for the target and
its "<target>:<labels>" children."""
out: dict[str, list[Sample]] = {}
for row in latest:
if row.resource_name == target:
labels = ""
elif row.resource_name.startswith(target + ":"):
labels = row.resource_name[len(target) + 1:]
else:
continue
out.setdefault(row.metric_name, []).append(Sample(
row.resource_name, row.metric_name, labels, dict(label_pairs(labels)), row.value,
getattr(row, "recorded_at", None)))
return out
def pick(index: dict[str, list[Sample]], metric: str, where: dict[str, str]) -> list[Sample]:
return [s for s in index.get(metric, []) if all(s.pairs.get(k) == v for k, v in where.items())]
def shown(metric: str, unit: str, value: float | None) -> dict:
"""{text, moment} for a value, in `unit` or else as its name says. A
count reads with thousands separated: 93,310."""
if unit == "count" and value is not None:
return {"text": f"{value:,.0f}" if value == int(value) else f"{value:,.2f}", "moment": None}
return shown_value(f"x{UNITS[unit]}" if unit else metric, value)
def _worst(tones) -> str:
tones = [t for t in tones if t != "dim"]
return min(tones, key=TONE_ORDER.__getitem__) if tones else "ok"
def _ordered(rows: list[Row]) -> list[Row]:
return sorted(rows, key=lambda r: (TONE_ORDER[r.tone], r.name.lower()))
# ── Blocks ───────────────────────────────────────────────────────────────────
def _same_row(index, metric: str, labels: str, name_label: str, name: str) -> list[Sample]:
"""The series of `metric` belonging to a state set's row: the same labels,
or else the same value of the label that names it."""
match = [s for s in index.get(metric, []) if s.labels == labels]
if not match and name:
match = [s for s in index.get(metric, []) if s.pairs.get(name_label) == name]
return match
def _overdue(spec: dict, index, labels: str, name: str, now: datetime) -> tuple[str, float] | None:
"""(tone, seconds past) when the row's crit_past or warn_past deadline has passed."""
for level in ("crit", "warn"):
metric = spec[f"{level}_past"]
if not metric:
continue
match = _same_row(index, metric, labels, spec["name"], name)
if len(match) == 1 and match[0].value > 0:
past = now.timestamp() - match[0].value
if past > 0:
return level, past
return None
def _states(block: Block, index, now) -> BlockView:
spec = block.spec
view = BlockView("states", block.title, columns=[c["label"] for c in spec["columns"]])
found = pick(index, spec["metric"], spec["where"])
folded = fold_states(spec["metric"], [{"labels": s.labels, "value": s.value, "resource": s.resource}
for s in found])
if not found:
view.note = f"Nothing reported for {spec['metric']}."
elif folded is None:
view.note = (f"{spec['metric']} isn't a state set: it should have one series per state, "
"the current one 1 and the rest 0.")
for f in folded or []:
pairs = dict(label_pairs(f["labels"]))
name = pairs.get(spec["name"], "") if spec["name"] else ""
current = [s for s in f["state"].split(", ") if s]
tones = []
for state in current:
if state in spec["healthy"]:
tones.append("ok")
elif state in spec["quiet"]:
tones.append("dim")
elif state in spec["warning"]:
tones.append("warn")
else:
tones.append("crit")
tone = (min(tones, key=TONE_ORDER.__getitem__) if tones else "warn")
why = "" if tone in ("ok", "dim") else (f"is {f['state']}" if current else "reports no state")
overdue = _overdue(spec, index, f["labels"], name, now) if tone == "ok" else None
if overdue:
tone, why = overdue[0], f"is {f['state']}, {human_duration(overdue[1])} overdue"
row = Row(name=name or f["labels"] or block.title, tone=tone, text=f["state"] or "—", why=why,
resource=f["resource"], metric=spec["metric"])
for col in spec["columns"]:
match = _same_row(index, col["metric"], f["labels"], spec["name"], name)
row.cells.append(shown(col["metric"], col["unit"], match[0].value if len(match) == 1 else None))
view.rows.append(row)
view.rows = _ordered(view.rows)
return view
def _limit_text(lim: Limit, metric: str, unit: str) -> str:
return f"{lim.value:g}%" if lim.share else (shown(metric, unit, lim.value)["text"] or f"{lim.value:g}")
def _judge(item: dict, value: float, of: float | None) -> tuple[str, str]:
for level in ("crit", "warn"):
for side in ("above", "below"):
lim = item[f"{level}_{side}"]
if lim is None:
continue
if lim.share:
if not of:
continue
x = value / of * 100
else:
x = value
if (x > lim.value) if side == "above" else (x < lim.value):
return level, f"{side} {_limit_text(lim, item['metric'], item['unit'])}"
return "ok", ""
def _rows_for(item: dict, index) -> list[tuple[str, Sample | None, str]]:
"""The series an item shows: one per value of its `name` label, or the one
series it names. (name, sample, problem)."""
found = pick(index, item["metric"], item["where"])
if item["name"]:
named = [(s.pairs.get(item["name"], "") or item["label"], s, "") for s in found]
return named or [(item["label"], None, "not reported")]
if len(found) == 1:
return [(item["label"], found[0], "")]
if not found:
return [(item["label"], None, "not reported")]
return [(item["label"], None, f"{len(found)} series match; narrow it with `where` or `name`")]
def _values(block: Block, index, now) -> tuple[BlockView, list]:
view, wants = BlockView("values", block.title), []
for item in block.spec["items"]:
for name, s, problem in _rows_for(item, index):
if s is None:
view.rows.append(Row(name=name, tone="warn", why=problem, text="—", metric=item["metric"]))
continue
row = Row(name=name, resource=s.resource, metric=s.metric, **shown(s.metric, item["unit"], s.value))
of = None
if item["of"]:
whole = [o for o in index.get(item["of"], []) if o.labels == s.labels]
if len(whole) == 1 and whole[0].value:
of = whole[0].value
row.bar = max(0.0, min(1.0, s.value / of))
row.text = f"{row.text} of {shown(item['of'], item['unit'], of)['text']}"
row.tone, row.why = _judge(item, s.value, of)
if item["sparkline"]:
wants.append((s.resource, s.metric))
view.rows.append(row)
view.rows = _ordered(view.rows)
return view, wants
def _times(block: Block, index, now) -> BlockView:
view = BlockView("times", block.title)
for item in block.spec["items"]:
for name, s, problem in _rows_for(item, index):
if s is None:
view.rows.append(Row(name=name, tone="warn", why=problem, text="—", metric=item["metric"]))
continue
got = shown_value("x_timestamp_seconds", s.value)
row = Row(name=name, resource=s.resource, metric=s.metric, **got)
if row.moment is None:
if item["warn_after"] or item["crit_after"]:
row.tone, row.why = "warn", "never"
else:
age = (now - row.moment).total_seconds()
for level in ("crit", "warn"):
after = item[f"{level}_after"]
if after and age > after:
row.tone, row.why = level, f"over {human_duration(after)} ago"
break
view.rows.append(row)
view.rows = _ordered(view.rows)
return view
def _table(block: Block, index, now) -> BlockView:
spec = block.spec
found = pick(index, spec["metric"], spec["where"])
rows: dict[str, dict[str, float]] = {}
columns: list[str] = []
for s in found:
r, c = s.pairs.get(spec["rows"]), s.pairs.get(spec["columns"])
if r is None or c is None:
continue
rows.setdefault(r, {})[c] = s.value
if c not in columns:
columns.append(c)
columns.sort()
view = BlockView("table", block.title, columns=columns)
if not rows:
view.note = (f"Nothing reported for {spec['metric']} with labels "
f"{spec['rows']} and {spec['columns']}.")
for name in sorted(rows, key=str.lower):
view.rows.append(Row(name=name, cells=[shown(spec["metric"], spec["unit"], rows[name].get(c))
for c in columns]))
return view
def _info(block: Block, index, now) -> BlockView:
spec = block.spec
view = BlockView("info", block.title)
found = sorted(pick(index, spec["metric"], spec["where"]), key=lambda s: s.labels)
if not found:
view.note = f"Nothing reported for {spec['metric']}."
for s in found:
keys = spec["labels"] or list(s.pairs)
view.rows.append(Row(name=" · ".join(f"{k} {s.pairs[k]}" for k in keys if s.pairs.get(k)),
resource=s.resource, metric=s.metric))
return view
_DRAW = {"states": _states, "times": _times, "table": _table, "info": _info}
def render(template: Template, target: str, latest, now: datetime,
history: dict[tuple[str, str], list[float]] | None = None) -> View:
"""The view of `template` over a target's newest values. `history` gives
the sparkline points the first pass asked for in `wants`."""
index = samples(target, latest)
blocks, wants = [], []
for block in template.blocks:
if block.kind == "values":
view, want = _values(block, index, now)
wants += want
else:
view = _DRAW[block.kind](block, index, now)
if history:
for row in view.rows:
row.history = history.get((row.resource, row.metric))
view.tone = "warn" if view.note else _worst(r.tone for r in view.rows)
blocks.append(view)
note = ""
if template.expects and not any(m.startswith(template.expects) for m in index):
note = (f"{target} reports no {template.expects}… metrics: is “{template.name}” "
"the template for it, or is the target filtering them out?")
problems = [(b, r) for b in blocks for r in b.rows if r.tone in ("crit", "warn")]
problems.sort(key=lambda p: TONE_ORDER[p[1].tone])
empty = [b for b in blocks if b.note]
tone = _worst([b.tone for b in blocks] + (["warn"] if note else []))
if note:
verdict = note
elif len(problems) + len(empty) == 0:
verdict = "All well."
else:
first = (f"{problems[0][0].title}: {problems[0][1].name} {problems[0][1].why}." if problems
else f"{empty[0].title}: {empty[0].note}")
count = len(problems) + len(empty)
verdict = first if count == 1 else f"{count} need a look. {first}"
return View(tone=tone, verdict=verdict, blocks=blocks, wants=wants, note=note)
plugins/prometheus/templates/prometheus/_preview.html
-14
View File
@@ -1,14 +0,0 @@
{# prometheus/_preview.html — the template editor's live preview (#5692):
what's typed, checked; when it passes, drawn over the chosen target as
the dashboard widget would draw it (widget.html, the same fragment). #}
{% if problems %}
<div class="w-verdict tone-warn">{{ problems|length }} problem{{ "s" if problems|length != 1 }} to fix before it draws.</div>
<ul class="tpl-problems">{% for p in problems %}<li>{{ p }}</li>{% endfor %}</ul>
{% elif empty %}
<p class="field-hint">Paste or write a template, and it's drawn here as you type.</p>
{% elif view is none %}
<div class="w-verdict tone-ok">“{{ template.name }}” passes the checker.</div>
<p class="field-hint">Add a Prometheus target to see it drawn over real values.</p>
{% else %}
{% include "prometheus/widget.html" %}
{% endif %}
plugins/prometheus/templates/prometheus/_tabs.html
-7
View File
@@ -1,7 +0,0 @@
{# The Prometheus section's tabs: its targets, and the widget templates the
dashboard widget draws with (#5692). Call in the section_tabs block. #}
{% from "_macros.html" import section_tabs %}
{% macro prometheus_tabs(active) -%}
{{ section_tabs([("targets", "Targets", "/plugins/prometheus/"),
("templates", "Widget templates", "/plugins/prometheus/templates/")], active, "Prometheus") }}
{%- endmacro %}
plugins/prometheus/templates/prometheus/_test.html
-18
View File
@@ -1,18 +0,0 @@
{# What Test found (milestone 506): how many series the address hands over
and which would be stored with the filters typed, or why it got nothing. #}
{% if found.error %}
<p class="tone-crit" role="alert" style="margin:0.75rem 0 0;">{{ found.error }}</p>
{% else %}
<div style="margin-top:0.75rem;">
<p class="tone-ok" style="margin:0 0 0.4rem;">{{ url }} answered with {{ found.families }} metric famil{{ "y" if found.families == 1 else "ies" }}.</p>
<p class="field-hint" style="margin:0 0 0.5rem;">With what's typed here, {{ found.kept }} series would be stored
{%- if found.dropped %}, {{ found.dropped }} left out by the filters{% endif %}
{%- if found.capped %}, and <span class="tone-warn">{{ found.capped }} over the cap</span>{% endif %}.</p>
{% if found.names %}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(18rem,1fr));gap:0.15rem 0.9rem;font-size:0.82rem;">
{% for name, n in found.names %}<span style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ name }}"><span class="mono">{{ name }}</span>{% if n > 1 %} <span class="tone-dim">×{{ n }}</span>{% endif %}</span>{% endfor %}
</div>
{% if found.more %}<p class="field-hint" style="margin:0.4rem 0 0;">and {{ found.more }} more.</p>{% endif %}
{% endif %}
</div>
{% endif %}
plugins/prometheus/templates/prometheus/form.html
-113
View File
@@ -1,113 +0,0 @@
{# plugins/prometheus/templates/prometheus/form.html — add or edit a scrape
target (milestone 506): where its /metrics is, its token, how often, and
what to keep, with Test to scrape it once before saving. The token is
never sent back to the browser; a blank token field keeps the stored one. #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header %}
{% block title %}{{ ("Edit " ~ target.name) if target else "Add Prometheus target" }} — Steward{% endblock %}
{% from "prometheus/_tabs.html" import prometheus_tabs %}
{% block section_tabs %}{{ prometheus_tabs("targets") }}{% endblock %}
{% block breadcrumb %}
{%- if target -%}
{{ crumbs([("Prometheus", "/plugins/prometheus/"), (target.name, "/plugins/prometheus/targets/" ~ target.id), ("Edit", "")]) }}
{%- else -%}
{{ crumbs([("Prometheus", "/plugins/prometheus/"), ("Add target", "")]) }}
{%- endif -%}
{% endblock %}
{% block content %}
{{ page_header(("Edit " ~ target.name) if target else "Add Prometheus target") }}
<div class="form-page">
{% if error %}<p class="field-error" role="alert">{{ error }}</p>{% endif %}
<form method="post" id="prom-form" action="{{ '/plugins/prometheus/targets/' ~ target.id ~ '/edit' if target else '/plugins/prometheus/targets/new' }}">{{ csrf_field() }}
{% if target %}<input type="hidden" name="target_id" value="{{ target.id }}">{% endif %}
<section class="card form-section" aria-labelledby="prom-where">
<h2 id="prom-where" class="section-title">Endpoint</h2>
<div class="form-pair">
<div class="field">
<label class="field-label" for="prom-name">Name</label>
<input class="input" id="prom-name" name="name" required maxlength="100" value="{{ v.name or '' }}" placeholder="Archives" autocomplete="off">
<span class="field-hint">How its series are named in Steward: alert rules and charts show it.</span>
</div>
<div class="field">
<label class="field-label" for="prom-url">Metrics address</label>
<input class="input mono" id="prom-url" name="url" required maxlength="500" value="{{ v.url or '' }}" placeholder="http://archives:8081/metrics" autocomplete="off">
</div>
<div class="field">
<label class="field-label" for="prom-token">Bearer token</label>
<input class="input" id="prom-token" name="token" type="password" autocomplete="new-password"
placeholder="{{ 'kept; type to change' if v.has_token else 'none' }}">
<span class="field-hint">Sent as "Authorization: Bearer …". Blank {{ "keeps the saved one" if v.has_token else "sends none" }}.</span>
{% if v.has_token %}
<label style="display:flex;gap:0.5rem;align-items:center;margin-top:0.3rem;cursor:pointer;">
<input type="checkbox" name="clear_token" value="on"> <span class="field-hint">Remove the saved token</span>
</label>
{% endif %}
</div>
<div class="field">
<label class="field-label" for="prom-interval">Scrape every (seconds)</label>
<input class="input" id="prom-interval" name="interval_seconds" type="number" min="10" max="3600" value="{{ v.interval_seconds or 60 }}">
</div>
</div>
<div style="display:flex;gap:1.5rem;flex-wrap:wrap;margin-bottom:0.9rem;">
<label style="display:flex;gap:0.5rem;align-items:center;cursor:pointer;">
<input type="checkbox" name="enabled" value="on"{% if v.enabled %} checked{% endif %}> Scrape this target
</label>
<label style="display:flex;gap:0.5rem;align-items:center;cursor:pointer;">
<input type="checkbox" name="verify_tls" value="on"{% if v.verify_tls %} checked{% endif %}> Check its HTTPS certificate
</label>
</div>
<div style="display:flex;align-items:center;gap:0.75rem;flex-wrap:wrap;">
<button type="button" class="btn btn-sm btn-ghost"
hx-post="/plugins/prometheus/targets/test" hx-include="#prom-form" hx-target="#prom-test"
hx-indicator="#prom-testing">Test</button>
<span id="prom-testing" class="field-hint htmx-indicator">Scraping…</span>
<span class="field-hint">Scrapes the address once with what's typed here, filters included, and says what would be stored. Nothing is saved.</span>
</div>
<div id="prom-test" aria-live="polite"></div>
</section>
<section class="card form-section" aria-labelledby="prom-what">
<h2 id="prom-what" class="section-title">What to keep</h2>
<p class="field-hint" style="margin-top:0;">The endpoint hands over everything; Steward stores what you choose. Patterns match the endpoint's metric names, with <span class="mono">*</span> for any run of characters.</p>
<div class="form-pair">
<div class="field">
<label class="field-label" for="prom-keep">Keep</label>
<textarea class="input mono" id="prom-keep" name="keep" rows="4" placeholder="fabledarchives_*">{{ v.keep or '' }}</textarea>
<span class="field-hint">One a line. Empty keeps everything.</span>
</div>
<div class="field">
<label class="field-label" for="prom-drop">Leave out</label>
<textarea class="input mono" id="prom-drop" name="drop" rows="4" placeholder="go_*&#10;process_*">{{ v.drop or '' }}</textarea>
<span class="field-hint">One a line. Wins over Keep.</span>
</div>
<div class="field">
<label class="field-label" for="prom-cap">Series cap</label>
<input class="input" id="prom-cap" name="series_cap" type="number" min="1" max="10000" value="{{ v.series_cap or 1000 }}">
<span class="field-hint">The most series stored from one scrape. Past it, the rest are counted and shown, not stored.</span>
</div>
</div>
<p class="field-hint" style="margin:0;">Gauges are stored as they are. Counters are stored as a rate per second, named <span class="mono">…_per_sec</span>, from the second scrape on. Histogram buckets and summary quantiles aren't stored; their <span class="mono">_sum</span> and <span class="mono">_count</span> are, as rates.</p>
</section>
<div class="form-actions">
<button type="submit" class="btn">{{ "Save" if target else "Add target" }}</button>
<a href="{{ '/plugins/prometheus/targets/' ~ target.id if target else '/plugins/prometheus/' }}" class="btn btn-ghost">Cancel</a>
</div>
</form>
{% if target %}
<section class="card danger-zone" aria-labelledby="prom-delete">
<div>
<h2 id="prom-delete" class="section-title">Delete this target</h2>
<p class="field-hint">Steward stops scraping it. Its past series age out with the rest of the history.</p>
</div>
<form method="post" action="/plugins/prometheus/targets/{{ target.id }}/delete"
data-confirm="Delete Prometheus target {{ target.name }}? Steward stops scraping it.">{{ csrf_field() }}
<button class="btn btn-danger" type="submit">Delete target</button>
</form>
</section>
{% endif %}
</div>
{% endblock %}
plugins/prometheus/templates/prometheus/index.html
-61
View File
@@ -1,61 +0,0 @@
{# plugins/prometheus/templates/prometheus/index.html — every scrape target,
those not answering first, each saying what its newest scrape found: how
many series it stored, or why it got nothing (milestone 506). #}
{% extends "base.html" %}
{% from "_macros.html" import page_header, status_badge, stat_tile, empty_state %}
{% block title %}Prometheus — Steward{% endblock %}
{% from "prometheus/_tabs.html" import prometheus_tabs %}
{% block section_tabs %}{{ prometheus_tabs("targets") }}{% endblock %}
{% block content %}
{% set admin = session.user_role == 'admin' %}
{% call page_header("Prometheus targets", (targets|length ~ " target" ~ ("s" if targets|length != 1 else "")) if targets else none) %}
{% if admin %}<a class="btn btn-sm" href="/plugins/prometheus/targets/new">Add target</a>{% endif %}
{% endcall %}
{% if not targets %}
<div class="card">{{ empty_state("No targets yet. Add a service's /metrics address, and Steward scrapes it on a schedule so its numbers can be charted and alerted on.",
"Add target" if admin else none, "/plugins/prometheus/targets/new") }}</div>
{% endif %}
{% for t in targets %}
<section class="card" aria-labelledby="prom-{{ t.id }}">
<div class="host-health-head">
{% if t.reach == 'up' %}{{ status_badge('up', 'Answering') }}
{% elif t.reach == 'down' %}{{ status_badge('down', 'Not answering') }}
{% elif t.reach == 'stale' %}{{ status_badge('stale', 'Not scraped') }}
{% elif t.reach == 'off' %}{{ status_badge('stale', 'Turned off') }}
{% else %}{{ status_badge('pending', 'Not scraped yet') }}{% endif %}
<h2 id="prom-{{ t.id }}" class="section-title" style="margin:0;"><a href="/plugins/prometheus/targets/{{ t.id }}">{{ t.name }}</a></h2>
<span class="field-hint mono" style="min-width:0;max-width:28rem;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ t.url }}">{{ t.url }}</span>
<span class="page-actions" style="margin-left:auto;">
<a href="/plugins/prometheus/targets/{{ t.id }}" class="btn btn-sm btn-ghost">Series</a>
{% if admin %}<a href="/plugins/prometheus/targets/{{ t.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>{% endif %}
</span>
</div>
{% if t.reach == 'down' %}
<p class="tone-crit" style="margin:0 0 0.4rem;">{{ t.says }}</p>
<p class="field-hint" style="margin:0 0 0.6rem;">
{%- if t.ok_at %}Last answered {{ t.ok_at|ago }}.{% else %}It has never answered.{% endif %}
Tried {{ t.scraped_at|ago }}.</p>
{% elif t.reach in ('stale', 'off') %}
<p class="field-hint" style="margin:0 0 0.6rem;">{{ t.says }}{% if t.scraped_at %} Last scraped {{ t.scraped_at|ago }}.{% endif %}</p>
{% elif t.reach == 'pending' %}
<p class="field-hint" style="margin:0 0 0.6rem;">Not scraped yet: the first scrape runs within seconds of saving.</p>
{% endif %}
{% if t.reach == 'up' %}
<div class="stat-strip" style="margin-bottom:0;">
{{ stat_tile("Series stored", t.kept) }}
{{ stat_tile("Over the cap", t.capped, hint=("cap is " ~ t.cap) if t.capped else none) }}
{{ stat_tile("Left out by filters", t.dropped) }}
{{ stat_tile("Scrape time", (t.scrape_ms|round|int ~ " ms") if t.scrape_ms is not none else "—") }}
{{ stat_tile("Every", t.interval|duration) }}
</div>
{% if t.capped %}
<p class="tone-warn" style="margin:0.6rem 0 0;">{{ t.capped }} series weren't stored: past the cap of {{ t.cap }}, or a name too long to store. {% if admin %}<a href="/plugins/prometheus/targets/{{ t.id }}/edit">Narrow what's kept or raise the cap.</a>{% endif %}</p>
{% endif %}
{% endif %}
</section>
{% endfor %}
{% endblock %}
plugins/prometheus/templates/prometheus/series.html
-68
View File
@@ -1,68 +0,0 @@
{# plugins/prometheus/templates/prometheus/series.html — one stored series
of a target (milestone 506): its newest value and a chart over a chosen
range. A *_timestamp_seconds series charts as moments, not numbers. #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header %}
{% block title %}{{ metric }} — {{ target.name }} — Steward{% endblock %}
{% from "prometheus/_tabs.html" import prometheus_tabs %}
{% block section_tabs %}{{ prometheus_tabs("targets") }}{% endblock %}
{% block breadcrumb %}{{ crumbs([("Prometheus", "/plugins/prometheus/"), (target.name, "/plugins/prometheus/targets/" ~ target.id), (metric, "")]) }}{% endblock %}
{% block content %}
{% call page_header(metric, labels or none) %}
{% for h in ranges %}
<a href="?resource={{ resource|urlencode }}&metric={{ metric|urlencode }}&hours={{ h }}" class="range-btn{% if hours == h %} active{% endif %}"{% if hours == h %} aria-current="true"{% endif %}>{% if h < 168 %}{{ h }}h{% else %}7d{% endif %}</a>
{% endfor %}
{% endcall %}
<p class="page-verdict">Now: <strong>{% if now.moment %}{{ now.moment|ago }}{% else %}{{ now.text }}{% endif %}</strong></p>
<section class="card" aria-label="{{ metric }} over the last {{ hours }} hours">
<div style="position:relative;height:260px;">
<canvas id="prom-chart" role="img" aria-label="{{ metric }} over the last {{ hours }} hours"></canvas>
</div>
</section>
<p class="field-hint">To be told when it crosses a line, add an alert rule on <strong>Prometheus targets</strong> → <span class="mono">{{ resource }}</span> → <span class="mono">{{ metric }}</span>.
<a href="/alerts/rules/new">New alert rule</a></p>
<script>
(function () {
var points = {{ points_json | safe }};
var hours = {{ hours }};
var moments = {{ "true" if stamp_axis else "false" }};
var el = document.getElementById("prom-chart");
if (points.length < 2) {
el.parentElement.innerHTML = '<p class="field-hint" style="margin:0;">Not enough samples yet for a chart.</p>';
return;
}
function stamp(iso) {
var d = new Date(iso);
return hours > 24
? d.toLocaleDateString([], { weekday: "short", day: "numeric" }) + " " + d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false })
: d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false });
}
function moment(v) {
return new Date(v * 1000).toLocaleString([], { day: "numeric", month: "short", hour: "2-digit", minute: "2-digit", hour12: false });
}
new Chart(el.getContext("2d"), {
type: "line",
data: { labels: points.map(function (p) { return p[0]; }),
datasets: [{ data: points.map(function (p) { return p[1]; }), borderColor: stewardColor("--blue"),
backgroundColor: "transparent", borderWidth: 1.5, pointRadius: 0, tension: 0.2 }] },
options: {
responsive: true, maintainAspectRatio: false, animation: false,
interaction: { mode: "index", intersect: false },
plugins: { legend: { display: false },
tooltip: { callbacks: {
title: function (items) { return items.length ? new Date(points[items[0].dataIndex][0]).toLocaleString([], { hour12: false }) : ""; },
label: function (item) { return moments ? moment(item.parsed.y) : String(item.parsed.y); } } } },
scales: {
x: { ticks: { maxTicksLimit: 8, maxRotation: 0, font: { size: 10 },
callback: function (v) { return stamp(points[v][0]); } }, grid: { display: false } },
y: { ticks: { font: { size: 10 }, maxTicksLimit: 5,
callback: function (v) { return moments ? moment(v) : v; } } }
}
}
});
})();
</script>
{% endblock %}
plugins/prometheus/templates/prometheus/target.html
-74
View File
@@ -1,74 +0,0 @@
{# plugins/prometheus/templates/prometheus/target.html — one scrape target
(milestone 506): what its newest scrape found, then every series it
stored, grouped by metric, each with its labels and newest value and a
link to its chart. A state set reads as one row per thing it describes,
naming the current state (#5689). #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header, status_badge, stat_tile %}
{% block title %}Prometheus — {{ target.name }} — Steward{% endblock %}
{% from "prometheus/_tabs.html" import prometheus_tabs %}
{% block section_tabs %}{{ prometheus_tabs("targets") }}{% endblock %}
{% block breadcrumb %}{{ crumbs([("Prometheus", "/plugins/prometheus/"), (target.name, "")]) }}{% endblock %}
{% block content %}
{% set admin = session.user_role == 'admin' %}
{% call page_header(target.name, target.url) %}
{% if admin %}<a href="/plugins/prometheus/targets/{{ target.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>{% endif %}
{% endcall %}
<p class="page-verdict">
{% if target.reach == 'up' %}{{ status_badge('up', 'Answering') }} scraped {{ target.scraped_at|ago }}, every {{ target.interval|duration }}
{% elif target.reach == 'down' %}{{ status_badge('down', 'Not answering') }} <span class="tone-crit">{{ target.says }}</span>
{% if target.ok_at %}Last answered {{ target.ok_at|ago }}; the values below are from then.{% else %}It has never answered.{% endif %}
{% elif target.reach == 'stale' %}{{ status_badge('stale', 'Not scraped') }} {{ target.says }}
{% elif target.reach == 'off' %}{{ status_badge('stale', 'Turned off') }} {{ target.says }}
{% else %}{{ status_badge('pending', 'Not scraped yet') }} The first scrape runs within seconds of saving.{% endif %}
</p>
{% if target.capped %}
<p class="tone-warn">{{ target.capped }} series weren't stored on the last scrape: past the cap of {{ target.cap }}, or a name too long to store.
{% if admin %}<a href="/plugins/prometheus/targets/{{ target.id }}/edit">Narrow what's kept or raise the cap.</a>{% endif %}</p>
{% endif %}
{% if target.reach != 'pending' %}
<div class="stat-strip">
{{ stat_tile("Series stored", target.kept) }}
{{ stat_tile("Over the cap", target.capped, state="degraded" if target.capped else none) }}
{{ stat_tile("Left out by filters", target.dropped) }}
{{ stat_tile("Scrape time", (target.scrape_ms|round|int ~ " ms") if target.scrape_ms is not none else "—") }}
</div>
{% endif %}
<p class="field-hint">Keeping {{ target.keep|join(", ") if target.keep else "everything" }}{% if target.drop %}, leaving out {{ target.drop|join(", ") }}{% endif %}.
Alert rules can watch any of these under <strong>Prometheus targets</strong>.</p>
<section class="card" aria-labelledby="prom-series">
<div style="display:flex;align-items:center;gap:0.75rem;flex-wrap:wrap;margin-bottom:0.5rem;">
<h2 id="prom-series" class="section-title" style="margin:0;">Series <span class="tone-muted">{{ total }}</span></h2>
<form method="get" style="margin-left:auto;display:flex;gap:0.5rem;">
<label class="sr-only" for="prom-q">Find a metric</label>
<input class="input" id="prom-q" name="q" value="{{ q }}" placeholder="Find a metric" style="max-width:16rem;">
{% if q %}<a class="btn btn-sm btn-ghost" href="/plugins/prometheus/targets/{{ target.id }}">Clear</a>{% endif %}
</form>
</div>
{% if not groups %}
<p class="field-hint" style="margin:0;">{% if q %}No stored metric matches “{{ q }}”.{% elif target.reach == 'pending' %}Nothing stored yet.{% else %}Nothing stored: the filters leave every series out, or the target hasn't answered.{% endif %}</p>
{% else %}
<table class="table">
<thead><tr><th scope="col">Metric</th><th scope="col">Labels</th><th scope="col" style="text-align:right;">Now</th></tr></thead>
<tbody>
{% for g in groups %}
{% for r in g.rows %}
<tr>
{% if loop.first %}<td class="mono" rowspan="{{ g.rows|length }}" style="vertical-align:top;overflow-wrap:anywhere;">{{ g.metric }}
{%- if g.states %}<div class="cell-sub">a state set: the current state is shown</div>{% endif %}</td>{% endif %}
<td class="mono" style="overflow-wrap:anywhere;font-size:0.82rem;">{{ r.labels or "—" }}</td>
<td style="text-align:right;white-space:nowrap;">
<a href="/plugins/prometheus/targets/{{ target.id }}/series?resource={{ r.resource|urlencode }}&metric={{ g.metric|urlencode }}"
{%- if r.states %} title="One of: {{ r.states|join(', ') }}"{% endif %}>{% if r.moment %}{{ r.moment|ago }}{% else %}{{ r.text }}{% endif %}</a>
</td>
</tr>
{% endfor %}
{% endfor %}
</tbody>
</table>
{% endif %}
</section>
{% endblock %}
plugins/prometheus/templates/prometheus/template.html
-92
View File
@@ -1,92 +0,0 @@
{# plugins/prometheus/templates/prometheus/template.html — one widget
template (#5692): its YAML beside a live preview over a chosen target,
drawn as the dashboard widget draws it. An admin edits their own here,
or starts one by pasting or uploading; a built-in one is read-only and
can be duplicated. The preview saves nothing. #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header %}
{% from "prometheus/_tabs.html" import prometheus_tabs %}
{% block title %}{{ title }} — Widget templates — Steward{% endblock %}
{% block section_tabs %}{{ prometheus_tabs("templates") }}{% endblock %}
{% block breadcrumb %}{{ crumbs([("Prometheus", "/plugins/prometheus/"), ("Widget templates", "/plugins/prometheus/templates/"), (title, "")]) }}{% endblock %}
{% block content %}
{% set admin = session.user_role == 'admin' %}
{% call page_header(title, ("Built in, read-only" if entry.bundled else "Your own") if entry else none) %}
{% if entry %}
<a class="btn btn-sm btn-ghost" href="/plugins/prometheus/templates/{{ entry.key }}/export">Export</a>
{% if admin %}<a class="btn btn-sm btn-ghost" href="/plugins/prometheus/templates/new?from={{ entry.key|urlencode }}">Duplicate</a>{% endif %}
{% endif %}
{% endcall %}
{% if problems %}
<div class="field-error" role="alert">
{{ "Not saved. " if saving }}{{ problems|length }} problem{{ "s" if problems|length != 1 }} to fix:
<ul class="tpl-problems">{% for p in problems %}<li>{{ p }}</li>{% endfor %}</ul>
</div>
{% endif %}
{% if note %}<p class="field-hint">{{ note }}</p>{% endif %}
{% if entry and entry.bundled %}
<p class="field-hint">Ships with Steward and changes with it. To change it, duplicate it and edit the copy.</p>
{% endif %}
{% if used %}<p class="field-hint">{{ used }} dashboard widget{{ "s draw" if used != 1 else " draws" }} with it{{ "; saving changes them too" if editable }}.</p>{% endif %}
<div class="tpl-editor">
<form method="post" id="tpl-form"
action="{{ '/plugins/prometheus/templates/custom:' ~ entry.id ~ '/edit' if entry else '/plugins/prometheus/templates/new' }}">{{ csrf_field() }}
<label class="field-label" for="tpl-text">YAML</label>
<textarea class="input mono tpl-text" id="tpl-text" name="text" spellcheck="false" wrap="off"
{%- if not editable %} readonly{% endif %}
placeholder="Paste a template here, or write one: name, version and blocks. The guide is docs/plugins/prometheus-widgets.md.">{{ text }}</textarea>
{% if editable %}
<div class="form-actions">
<button type="submit" class="btn">{{ "Save" if entry else "Save template" }}</button>
<a href="{{ '/plugins/prometheus/templates/' ~ entry.key if entry else '/plugins/prometheus/templates/' }}" class="btn btn-ghost">Cancel</a>
</div>
{% endif %}
</form>
<section class="card" aria-labelledby="tpl-preview-title">
<div class="tpl-preview-head">
<h2 id="tpl-preview-title" class="section-title">Preview</h2>
{% if targets %}
<label class="sr-only" for="tpl-target">Target</label>
<select class="input" id="tpl-target" name="target">
{% for t in targets %}<option value="{{ t.value }}">{{ t.label }}</option>{% endfor %}
</select>
{% endif %}
</div>
<div id="tpl-preview" class="dash-cell-body" aria-live="polite"
hx-post="/plugins/prometheus/templates/preview"
hx-trigger="load, input changed delay:600ms from:#tpl-text, change from:#tpl-target"
hx-include="#tpl-text, #tpl-target">
<p class="field-hint">Loading…</p>
</div>
</section>
</div>
{% if editable and not entry %}
<section class="card form-section" aria-labelledby="tpl-upload">
<h2 id="tpl-upload" class="section-title">Or upload a file</h2>
<form method="post" action="/plugins/prometheus/templates/upload" enctype="multipart/form-data"
style="display:flex;gap:0.75rem;align-items:center;flex-wrap:wrap;">{{ csrf_field() }}
<label class="sr-only" for="tpl-file">Template file</label>
<input id="tpl-file" type="file" name="file" accept=".yaml,.yml,text/yaml,application/yaml">
<button type="submit" class="btn btn-sm btn-ghost">Read file</button>
<span class="field-hint">Opens it here to look over; nothing is saved until you press Save.</span>
</form>
</section>
{% endif %}
{% if editable and entry %}
<section class="card danger-zone" aria-labelledby="tpl-delete">
<div>
<h2 id="tpl-delete" class="section-title">Delete this template</h2>
<p class="field-hint">{% if used %}{{ used }} widget{{ "s" if used != 1 }} using it will say the template is gone.{% else %}No widget uses it.{% endif %} Export it first to keep a copy.</p>
</div>
<form method="post" action="/plugins/prometheus/templates/custom:{{ entry.id }}/delete"
data-confirm="Delete widget template {{ entry.name }}?{% if used %} {{ used }} widget{{ 's' if used != 1 }} use it.{% endif %}">{{ csrf_field() }}
<button class="btn btn-danger" type="submit">Delete template</button>
</form>
</section>
{% endif %}
{% endblock %}
plugins/prometheus/templates/prometheus/templates.html
-50
View File
@@ -1,50 +0,0 @@
{# plugins/prometheus/templates/prometheus/templates.html — the widget
templates the Prometheus dashboard widget can draw with (#5692): the ones
built in and the ones copied in here. One that no longer passes the
checker comes first, saying why: its widgets say so too. #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header %}
{% from "prometheus/_tabs.html" import prometheus_tabs %}
{% block title %}Widget templates — Prometheus — Steward{% endblock %}
{% block section_tabs %}{{ prometheus_tabs("templates") }}{% endblock %}
{% block breadcrumb %}{{ crumbs([("Prometheus", "/plugins/prometheus/"), ("Widget templates", "")]) }}{% endblock %}
{% block content %}
{% set admin = session.user_role == 'admin' %}
{% set broken = templates|selectattr("problem")|list %}
{% set own = templates|rejectattr("bundled")|list %}
{% call page_header("Widget templates") %}
{% if admin %}<a class="btn btn-sm" href="/plugins/prometheus/templates/new">New template</a>{% endif %}
{% endcall %}
<p class="page-verdict">
{%- if broken|length == 1 %}<span class="tone-warn">1 template doesn't pass the checker; widgets drawn with it say so.</span>
{%- elif broken %}<span class="tone-warn">{{ broken|length }} templates don't pass the checker; widgets drawn with them say so.</span>
{%- else %}{{ templates|length }} template{{ "s" if templates|length != 1 }}: {{ templates|length - own|length }} built in, {{ own|length }} of your own.{% endif %}
</p>
<p class="field-hint">A template says which of a target's series the Prometheus dashboard widget shows, and how. Built-in ones can be copied and changed; your own you write here, or paste or upload one someone shared.</p>
<section class="card">
<table class="table">
<thead><tr><th scope="col">Template</th><th scope="col">Version</th><th scope="col">Used by</th><th scope="col">Changed</th><th scope="col"><span class="sr-only">Actions</span></th></tr></thead>
<tbody>
{% for t in templates %}
<tr>
<td>
<a href="/plugins/prometheus/templates/{{ t.key }}">{{ t.name }}</a>
{% if t.bundled %} <span class="tone-muted">built in</span>{% endif %}
{%- if t.problem %}<div class="cell-sub tone-warn" title="{{ t.problem }}">{{ t.problem }}</div>
{%- elif t.description %}<div class="cell-sub" title="{{ t.description }}">{{ t.description }}</div>{% endif %}
</td>
<td>{{ t.version if t.version is not none else "—" }}</td>
<td>{% if t.used %}{{ t.used }} widget{{ "s" if t.used != 1 }}{% else %}<span class="tone-muted">none</span>{% endif %}</td>
<td>{% if t.updated_at %}{{ t.updated_at|ago }}{% else %}<span class="tone-muted">with Steward</span>{% endif %}</td>
<td style="text-align:right;white-space:nowrap;">
<a class="btn btn-sm btn-ghost" href="/plugins/prometheus/templates/{{ t.key }}/export">Export</a>
{% if admin %}<a class="btn btn-sm btn-ghost" href="/plugins/prometheus/templates/new?from={{ t.key|urlencode }}">Duplicate</a>{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</section>
{% endblock %}
plugins/prometheus/templates/prometheus/widget.html
-70
View File
@@ -1,70 +0,0 @@
{# prometheus/widget.html — the Prometheus dashboard widget (#5691): one
target drawn by one template (template_view.render). A verdict naming the
worst thing, then each block: its problems first with why, figures in their
units beside a bar or the last day's bars, and the healthy rest packed after,
past REST rows as "+N more healthy". #}
{% from "_macros.html" import status_dot %}
{% set REST = 6 %}
{% set dot = {"crit": "crit", "warn": "warn", "ok": "ok", "dim": "paused"} %}
{% set tone = {"crit": "tone-crit", "warn": "tone-warn", "ok": "tone-ok", "dim": "tone-dim"} %}
{% macro shown(v) -%}
{%- if v.moment %}{{ v.moment|ago }}{% elif v.text %}{{ v.text }}{% else %}—{% endif -%}
{%- endmacro %}
{% if problem %}
<div class="w-verdict tone-warn">{{ problem }}</div>
{% else %}
{% set down = target.reach in ("down", "stale") %}
<div class="w-verdict {{ 'tone-crit' if target.reach == 'down' else tone[view.tone] }}">
{%- if target.reach == "down" %}{{ target.name }} isn't answering{% elif down %}{{ target.name }} isn't being scraped{% endif %}
{%- if down %}{% if target.ok_at %}; these values are from {{ target.ok_at|ago }}{% endif %}. {% endif -%}
{{ view.verdict }}
</div>
{% for b in view.blocks %}
<div class="w-section">{{ b.title }}</div>
{% if b.note %}
<div class="w-note tone-warn">{{ b.note }}</div>
{% elif b.kind == "info" %}
<div class="w-note">{% for r in b.rows %}{{ r.name }}{% if not loop.last %}; {% endif %}{% endfor %}</div>
{% elif b.kind == "table" %}
<table class="w-table">
<thead><tr><th></th>{% for c in b.columns %}<th>{{ c }}</th>{% endfor %}</tr></thead>
<tbody>
{% for r in b.rows %}
<tr><td class="w-name" title="{{ r.name }}">{{ r.name }}</td>{% for c in r.cells %}<td>{{ shown(c) }}</td>{% endfor %}</tr>
{% endfor %}
</tbody>
</table>
{% else %}
{% set problems = b.rows | selectattr("tone", "in", ["crit", "warn"]) | list %}
{% set rest = b.rows | rejectattr("tone", "in", ["crit", "warn"]) | list %}
{% set listed = problems + rest[:REST] %}
<table class="w-table">
{% if b.columns %}<thead><tr><th></th><th>Now</th>{% for c in b.columns %}<th class="w-wide-only">{{ c }}</th>{% endfor %}</tr></thead>{% endif %}
<tbody>
{% for r in listed %}
<tr>
<td class="w-name" title="{{ r.name }}">
{{- status_dot(dot[r.tone]) -}}
{{ r.name }}
{%- if r.why %}<span class="w-sub {{ tone[r.tone] }}">{{ r.why }}</span>{% endif %}
</td>
<td class="{{ tone[r.tone] if r.tone in ('crit', 'warn') }}">
{%- if r.history %}<span class="w-bars w-wide-only" aria-hidden="true" title="The last {{ spark_hours }}h, scaled to its own peak">
{%- for h in r.history %}<i style="height:{{ [h, 4]|max }}%"></i>{% endfor -%}
</span>{% endif -%}
{{ shown(r) }}
{%- if r.bar is not none %}<div class="meter w-wide-only" title="{{ (r.bar * 100)|round|int }}%"><div class="meter-fill meter-{{ r.tone if r.tone in ('crit', 'warn') else 'ok' }}" style="width:{{ (r.bar * 100)|round(1) }}%"></div></div>{% endif -%}
</td>
{% for c in r.cells %}<td class="w-wide-only">{{ shown(c) }}</td>{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
{% if rest|length > REST %}<div class="w-more">+{{ rest|length - REST }} more healthy</div>{% endif %}
{% endif %}
{% endfor %}
<div class="w-note">
{%- if session.user_id %}<a href="/plugins/prometheus/targets/{{ target.id }}">{{ target.name }}</a>{% else %}{{ target.name }}{% endif %}
{%- if target.scraped_at %}, scraped {{ target.scraped_at|ago }}{% endif %}.
</div>
{% endif %}
plugins/prometheus/widgets/fabled_archives.yaml
-68
View File
@@ -1,68 +0,0 @@
# Fabled Archives, from its /metrics on the LAN-only backup listener (:8081).
# Written against the series Steward stores for Archives (#5693); the format
# is docs/plugins/prometheus-widgets.md.
#
# Left out on purpose:
# - fabledarchives_section_up: a section Archives can't read stops reporting
# its series, and the rows here that read them then say "not reported".
# - the background queue (fabledarchives_jobs, *_job_attempts_*), HTTP
# requests and the audit counters: they belong on charts and alert rules,
# not on a glance.
name: Fabled Archives
description: Backup targets, replication jobs, storage, integrity and the version running.
version: 2
expects: fabledarchives_
blocks:
- states:
title: Backup targets
metric: fabledarchives_backup_target_state
name: target
healthy: [fresh]
quiet: [disabled, waiting]
warning: [late]
columns:
- {label: Last backup, metric: fabledarchives_backup_target_last_backup_timestamp_seconds}
- {label: Stored, metric: fabledarchives_backup_target_stored_bytes}
- states:
title: Replication
metric: fabledarchives_replication_job_state
name: job
# Archives ranks "running" above lateness: a run that never ends still
# reads "running". Its own deadlines say when it is overdue.
healthy: [ok, running]
warning: [late, warning, never]
warn_past: fabledarchives_replication_job_late_at_timestamp_seconds
crit_past: fabledarchives_replication_job_missing_at_timestamp_seconds
columns:
- {label: Last good copy, metric: fabledarchives_replication_job_last_success_timestamp_seconds}
- {label: Next run, metric: fabledarchives_replication_job_next_run_timestamp_seconds}
- values:
title: Storage and integrity
items:
- label: Disk free
metric: fabledarchives_data_disk_free_bytes
of: fabledarchives_data_disk_size_bytes
warn_below: 10%
crit_below: 5%
sparkline: true
- label: Files checked
metric: fabledarchives_files_checked
of: fabledarchives_files
unit: count
- label: Malware held
metric: fabledarchives_malware_held_files
unit: count
warn_above: 0
- label: Notification channels failing
metric: fabledarchives_notification_channels_failing
unit: count
warn_above: 0
- table:
title: Stored per person
metric: fabledarchives_person_stored_bytes
rows: person
columns: kind
- info:
title: Running
metric: fabledarchives_build_info
labels: [version, channel, commit]
plugins/snmp/__init__.py
-13
View File
@@ -11,7 +11,6 @@ _app: "Quart | None" = None
def setup(app: "Quart") -> None:
global _app
_app = app
from .models import SnmpDevice # noqa: F401 — registers the table
def get_scheduled_tasks() -> list:
@@ -27,15 +26,3 @@ def get_blueprint():
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (SNMP device readings).
return [{"label": "SNMP", "href": "/plugins/snmp/"}]
def get_alert_metrics() -> dict[str, list[str]]:
"""SNMP metric names are the OID labels each device is configured with, so
the list is empty here and the rule form offers the labels recorded so far."""
return {"snmp": []}
def get_audit_entities() -> dict:
"""Audit rows about a device name it and link to its page."""
from .models import SnmpDevice
return {"snmp_device": (SnmpDevice, "/plugins/snmp/devices/{}")}
plugins/snmp/migrations/__init__.py
View File
plugins/snmp/migrations/versions/__init__.py
View File
plugins/snmp/migrations/versions/snmp_001_devices.py
-49
View File
@@ -1,49 +0,0 @@
"""SNMP devices are rows edited on the SNMP page
Devices lived in plugin.yaml's config, which the UI could not edit, and a poll
that failed left no trace. A row per device holds what to ask and how, and the
newest poll's outcome (milestone 491).
Revision ID: snmp_001_devices
Revises: (none — a branch off core via depends_on)
Create Date: 2026-10-08
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "snmp_001_devices"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = "snmp"
depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
def upgrade() -> None:
op.create_table(
"snmp_devices",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("name", sa.String(100), nullable=False, unique=True),
sa.Column("address", sa.String(255), nullable=False),
sa.Column("port", sa.Integer, nullable=False, server_default="161"),
sa.Column("version", sa.String(3), nullable=False, server_default="2c"),
sa.Column("community", sa.Text, nullable=False, server_default=""),
sa.Column("v3_user", sa.String(100), nullable=False, server_default=""),
sa.Column("v3_auth", sa.String(16), nullable=False, server_default="none"),
sa.Column("v3_auth_key", sa.Text, nullable=False, server_default=""),
sa.Column("v3_priv", sa.String(16), nullable=False, server_default="none"),
sa.Column("v3_priv_key", sa.Text, nullable=False, server_default=""),
sa.Column("presets", sa.JSON, nullable=False, server_default="[]"),
sa.Column("interfaces", sa.String(255), nullable=False, server_default=""),
sa.Column("custom", sa.Text, nullable=False, server_default=""),
sa.Column("host_id", sa.String(36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("last_polled_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_ok_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text, nullable=True),
sa.Column("missing", sa.JSON, nullable=False, server_default="[]"),
)
def downgrade() -> None:
op.drop_table("snmp_devices")
plugins/snmp/models.py
-42
View File
@@ -1,42 +0,0 @@
# plugins/snmp/models.py
from __future__ import annotations
from datetime import datetime, timezone
from sqlalchemy import JSON, DateTime, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from steward.core.ids import time_ordered_id
from steward.models.base import Base
class SnmpDevice(Base):
"""A device Steward polls over SNMP, edited on the SNMP page.
Secrets (the community, the v3 keys) are stored encrypted. The last_*
columns are the newest poll's outcome, written by the poller, so a page
can say why a device isn't answering without asking the device itself.
"""
__tablename__ = "snmp_devices"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
address: Mapped[str] = mapped_column(String(255), nullable=False)
port: Mapped[int] = mapped_column(Integer, nullable=False, default=161)
version: Mapped[str] = mapped_column(String(3), nullable=False, default="2c")
community: Mapped[str] = mapped_column(Text, nullable=False, default="")
v3_user: Mapped[str] = mapped_column(String(100), nullable=False, default="")
v3_auth: Mapped[str] = mapped_column(String(16), nullable=False, default="none")
v3_auth_key: Mapped[str] = mapped_column(Text, nullable=False, default="")
v3_priv: Mapped[str] = mapped_column(String(16), nullable=False, default="none")
v3_priv_key: Mapped[str] = mapped_column(Text, nullable=False, default="")
presets: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
interfaces: Mapped[str] = mapped_column(String(255), nullable=False, default="")
custom: Mapped[str] = mapped_column(Text, nullable=False, default="")
host_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
last_polled_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_ok_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
missing: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
plugins/snmp/plugin.yaml
+45 -3
View File
@@ -1,7 +1,7 @@
# plugins/snmp/plugin.yaml
name: snmp
version: "2.0.0"
description: "SNMP polling for network devices — switches, routers, printers, UPSes, servers."
version: "1.0.0"
description: "SNMP polling for network devices — switches, routers, printers, UPSes, etc."
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
@@ -12,6 +12,48 @@ tags:
- network
- monitoring
# Devices are added, tested and edited on the SNMP page (/plugins/snmp/).
config:
poll_interval_seconds: 60
# Each device is polled independently. OIDs must resolve to numeric values.
#
# `host` is the SNMP poll target (IP/hostname we query). A device also shows
# up on a Steward Host's detail page when it maps to that host. By default
# that mapping is implicit: `host` is matched against the Host's address or
# name. To bind explicitly instead — e.g. when you poll by IP but the Host is
# recorded by DNS name, or for an SNMP-only switch/PDU/UPS — add ONE of:
# host_id: "<steward-host-uuid>" # exact Host.id match
# steward_host: "switch01" # Host name OR address (case-insensitive)
# An explicit binding is exclusive: the device then maps ONLY to that host,
# never implicitly to another by a coincidental address string.
devices:
- name: "core-switch"
host: "192.168.1.1"
port: 161
community: "public"
version: "2c" # "1", "2c", or "3"
oids:
- oid: "1.3.6.1.2.1.1.3.0"
label: "uptime_centisecs"
- oid: "1.3.6.1.2.1.2.2.1.10.1"
label: "if1_in_octets"
- oid: "1.3.6.1.2.1.2.2.1.16.1"
label: "if1_out_octets"
# Example: network UPS with SNMP management card (RFC 1628 UPS-MIB).
# upsBatteryStatus values: 1=unknown, 2=batteryNormal, 3=batteryLow, 4=batteryDepleted.
# Uncomment and point at your UPS to enable.
# - name: "ups"
# host: "192.168.1.10"
# port: 161
# community: "public"
# version: "2c"
# oids:
# - oid: "1.3.6.1.2.1.33.1.2.1.0"
# label: "battery_status"
# - oid: "1.3.6.1.2.1.33.1.2.2.0"
# label: "seconds_on_battery"
# - oid: "1.3.6.1.2.1.33.1.2.3.0"
# label: "runtime_minutes_remaining"
# - oid: "1.3.6.1.2.1.33.1.2.4.0"
# label: "battery_charge_pct"
# - oid: "1.3.6.1.2.1.33.1.3.3.1.3.1"
# label: "input_voltage_dV"
plugins/snmp/poller.py
+102 -198
View File
@@ -1,54 +1,19 @@
# plugins/snmp/poller.py
"""Ask an SNMP device for its readings, and say in words why it didn't answer.
"""
Asynchronous SNMP GET helper.
pysnmp 7.x (what the image installs, #5470) and the deprecated
pysnmp-lextudio 6.x put the asyncio API in different modules with different
names; _api() finds whichever is installed, so a dependency change can't
silently break polling. Without pysnmp at all the poll reports that instead
of raising.
Requires pysnmp-lextudio (the maintained pysnmp fork), bundled into the Docker
image via the `snmp` extra (`pip install .[snmp]`):
pip install 'steward[snmp]'
If pysnmp is not installed, poll_device() returns an empty dict and logs a
warning — SNMP polling is then simply disabled, nothing else breaks.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass, field
from types import SimpleNamespace
logger = logging.getLogger(__name__)
TIMEOUT_SECONDS = 5
# OIDs per GET. A device's whole list usually fits in one request; this keeps
# a long custom list under the size an agent will answer.
OIDS_PER_REQUEST = 24
AUTH_PROTOCOLS = {"none": "usmNoAuthProtocol", "md5": "usmHMACMD5AuthProtocol",
"sha": "usmHMACSHAAuthProtocol", "sha224": "usmHMAC128SHA224AuthProtocol",
"sha256": "usmHMAC192SHA256AuthProtocol", "sha384": "usmHMAC256SHA384AuthProtocol",
"sha512": "usmHMAC384SHA512AuthProtocol"}
PRIV_PROTOCOLS = {"none": "usmNoPrivProtocol", "des": "usmDESPrivProtocol", "3des": "usm3DESEDEPrivProtocol",
"aes128": "usmAesCfb128Protocol", "aes192": "usmAesCfb192Protocol",
"aes256": "usmAesCfb256Protocol"}
# A varbind value of one of these types means the agent has no such object.
_ABSENT = {"NoSuchObject", "NoSuchInstance", "EndOfMibView"}
@dataclass
class Credentials:
version: str = "2c"
community: str = ""
user: str = ""
auth: str = "none"
auth_key: str = ""
priv: str = "none"
priv_key: str = ""
@dataclass
class PollResult:
readings: dict[str, float] = field(default_factory=dict)
error: str | None = None # the device didn't answer, and why
missing: list[str] = field(default_factory=list) # keys it answered without
def _pysnmp_available() -> bool:
try:
@@ -58,27 +23,9 @@ def _pysnmp_available() -> bool:
return False
def _api() -> SimpleNamespace:
"""The installed pysnmp's asyncio API under one set of names."""
try:
from pysnmp.hlapi.v3arch import asyncio as hl
from pysnmp.hlapi.v3arch.asyncio import auth
return SimpleNamespace(hl=hl, auth=auth, get=hl.get_cmd, walk=hl.walk_cmd, async_transport=True)
except ImportError:
from pysnmp.hlapi import asyncio as hl
from pysnmp.hlapi.asyncio import auth
return SimpleNamespace(hl=hl, auth=auth, get=hl.getCmd, walk=hl.walkCmd, async_transport=False)
def _auth_data(api, creds: Credentials):
if creds.version == "3":
return api.hl.UsmUserData(
creds.user,
authKey=creds.auth_key or None,
privKey=creds.priv_key or None,
authProtocol=getattr(api.auth, AUTH_PROTOCOLS.get(creds.auth, "usmNoAuthProtocol")),
privProtocol=getattr(api.auth, PRIV_PROTOCOLS.get(creds.priv, "usmNoPrivProtocol")))
return api.hl.CommunityData(creds.community, mpModel=0 if creds.version == "1" else 1)
def _mp_model(version: str) -> int:
"""Map an SNMP version string to pysnmp's mpModel int (0 = v1, 1 = v2c)."""
return 0 if version == "1" else 1
def _close_engine(engine) -> None:
@@ -119,149 +66,106 @@ def _close_engine(engine) -> None:
return
def explain(error_indication, address: str, port: int, creds: Credentials) -> str:
"""pysnmp's error indication, said as what to check."""
text = str(error_indication)
low = text.lower()
if "timeout" in low or "no snmp response" in low:
who = (f"the v3 user “{creds.user}”" if creds.version == "3"
else "the community")
return (f"No reply from {address}:{port} within {TIMEOUT_SECONDS} s. Check the address and port, "
f"that SNMP is on, and {who}: a device ignores a request it doesn't accept.")
if "unknown" in low and "user" in low:
return f"The device doesn't know the v3 user “{creds.user}”."
if "digest" in low or "authentication" in low:
return "The device refused the v3 authentication: check the auth protocol and key."
if "decrypt" in low or "privacy" in low:
return "The device couldn't decrypt the request: check the privacy protocol and key."
if "not in time window" in low:
return "The device rejected the request as out of its time window; try again."
return f"SNMP error from {address}:{port}: {text}"
async def poll_device(
host: str,
port: int,
community: str,
version: str,
oids: list[dict],
) -> dict[str, float]:
"""Perform an SNMP GET for each OID and return ``{label: float_value}``.
Non-numeric OIDs (strings, etc.) are skipped. Returns an empty dict on any
error (unreachable host, wrong community, …) so a flaky device never breaks
the poll loop.
async def _transport(api, address: str, port: int):
if api.async_transport:
return await api.hl.UdpTransportTarget.create((address, port), timeout=TIMEOUT_SECONDS, retries=1)
return api.hl.UdpTransportTarget((address, port), timeout=TIMEOUT_SECONDS, retries=1)
async def poll_device(address: str, port: int, creds: Credentials,
oids: list[tuple[str, str]]) -> PollResult:
"""GET every (oid, key) of a device, a few dozen to a request.
Returns the numeric answers by key, the keys the device answered without
(no such object, or not a number), and an error in words when it didn't
answer at all. Never raises: a broken device must not stop the others.
pysnmp's HLAPI is asyncio-only as of v6. The import path moved between major
versions, so we support both rather than let a dependency bump silently
re-break polling:
• pysnmp-lextudio 6.2.x → ``pysnmp.hlapi.asyncio`` (``getCmd`` + a directly
constructed ``UdpTransportTarget``).
• canonical pysnmp 7.x → ``pysnmp.hlapi.v3arch.asyncio`` (``get_cmd`` + the
async ``UdpTransportTarget.create``).
"""
result = PollResult()
if not oids:
return result
if not _pysnmp_available():
result.error = "pysnmp isn't installed in this image, so SNMP polling is off."
return result
logger.warning("pysnmp not installed — SNMP polling disabled. "
"Install with: pip install 'steward[snmp]'")
return {}
api = _api()
engine = api.hl.SnmpEngine()
try:
try:
transport = await _transport(api, address, port)
except Exception as exc:
result.error = f"Can't reach {address}:{port}: {exc}"
return result
auth = _auth_data(api, creds)
# canonical pysnmp 7.x
from pysnmp.hlapi.v3arch.asyncio import (
CommunityData,
ContextData,
ObjectIdentity,
ObjectType,
SnmpEngine,
UdpTransportTarget,
get_cmd as _get_cmd,
)
_transport_is_async = True
except ImportError:
# pysnmp-lextudio 6.2.x
from pysnmp.hlapi.asyncio import (
CommunityData,
ContextData,
ObjectIdentity,
ObjectType,
SnmpEngine,
UdpTransportTarget,
getCmd as _get_cmd,
)
_transport_is_async = False
pending = list(oids)
while pending:
chunk, pending = pending[:OIDS_PER_REQUEST], pending[OIDS_PER_REQUEST:]
ei, es, eidx, var_binds = await api.get(
engine, auth, transport, api.hl.ContextData(),
*[api.hl.ObjectType(api.hl.ObjectIdentity(oid)) for oid, _ in chunk],
lookupMib=False)
if ei:
result.error = explain(ei, address, port, creds)
return result
if es:
# v1 fails the whole request on the first unknown OID: note it,
# and ask again for the rest.
bad = int(eidx) - 1 if eidx else -1
if 0 <= bad < len(chunk):
result.missing.append(chunk[bad][1])
pending = chunk[:bad] + chunk[bad + 1:] + pending
continue
result.error = f"The device answered with an error: {es.prettyPrint()}"
return result
# A GET answers in the order it asked.
for (_, key), (_, val) in zip(chunk, var_binds):
if type(val).__name__ in _ABSENT:
result.missing.append(key)
continue
engine = SnmpEngine()
# Always release the engine's UDP socket — see _close_engine for why.
try:
# Same host/port for every OID on this device, so build the transport once.
try:
if _transport_is_async:
transport = await UdpTransportTarget.create((host, port), timeout=5, retries=1)
else:
transport = UdpTransportTarget((host, port), timeout=5, retries=1)
except Exception as exc:
logger.debug("SNMP transport setup failed for %s:%s: %s", host, port, exc)
return {}
results: dict[str, float] = {}
for oid_cfg in oids:
oid = oid_cfg["oid"]
label = oid_cfg.get("label") or oid
scale = float(oid_cfg.get("scale", 1.0))
try:
error_indication, error_status, error_index, var_binds = await _get_cmd(
engine,
CommunityData(community, mpModel=_mp_model(version)),
transport,
ContextData(),
ObjectType(ObjectIdentity(oid)),
)
except Exception as exc:
logger.debug("SNMP GET %s@%s OID %s failed: %s", host, port, oid, exc)
continue
if error_indication:
logger.debug("SNMP error %s@%s OID %s: %s", host, port, oid, error_indication)
continue
if error_status:
logger.debug("SNMP status %s@%s OID %s: %s at %s",
host, port, oid, error_status.prettyPrint(),
error_index and var_binds[int(error_index) - 1][0] or "?")
continue
for _, val in var_binds:
try:
result.readings[key] = float(val)
results[label] = float(val) * scale
except (TypeError, ValueError):
result.missing.append(key)
if not result.readings and result.missing:
result.error = "The device answered but has none of these readings."
return result
except Exception as exc:
logger.warning("SNMP poll of %s:%s failed: %s", address, port, exc)
result.error = f"SNMP poll failed: {exc}"
return result
finally:
_close_engine(engine)
# Non-numeric type (e.g. OctetString description) — skip.
logger.debug("SNMP non-numeric value for %s label=%s: %r", oid, label, val)
SYS_DESCR, SYS_UPTIME, SYS_NAME = "1.3.6.1.2.1.1.1.0", "1.3.6.1.2.1.1.3.0", "1.3.6.1.2.1.1.5.0"
IF_DESCR, IF_NAME = "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.31.1.1.1.1"
INTERFACES_LISTED = 128
async def describe_device(address: str, port: int, creds: Credentials) -> dict:
"""What Test shows: {"error"} or {"name", "description", "uptime_cs",
"interfaces": [(number, name)]}. Asks the device directly, so only an
operator's explicit Test calls it, never a page load."""
if not _pysnmp_available():
return {"error": "pysnmp isn't installed in this image, so SNMP is off."}
api = _api()
engine = api.hl.SnmpEngine()
try:
try:
transport = await _transport(api, address, port)
except Exception as exc:
return {"error": f"Can't reach {address}:{port}: {exc}"}
auth = _auth_data(api, creds)
ei, es, _, var_binds = await api.get(
engine, auth, transport, api.hl.ContextData(),
*[api.hl.ObjectType(api.hl.ObjectIdentity(o)) for o in (SYS_NAME, SYS_DESCR, SYS_UPTIME)],
lookupMib=False)
if ei:
return {"error": explain(ei, address, port, creds)}
if es:
return {"error": f"The device answered with an error: {es.prettyPrint()}"}
values = [None if type(v).__name__ in _ABSENT else v for _, v in var_binds]
out = {"name": str(values[0]) if values[0] is not None else "",
"description": str(values[1]) if values[1] is not None else "",
"uptime_cs": float(values[2]) if values[2] is not None else None,
"interfaces": []}
# ifName where the device has it (IF-MIB), else ifDescr.
for column in (IF_NAME, IF_DESCR):
found: list[tuple[int, str]] = []
async for ei, es, _, rows in api.walk(engine, auth, transport, api.hl.ContextData(),
api.hl.ObjectType(api.hl.ObjectIdentity(column)),
lexicographicMode=False, lookupMib=False):
if ei or es:
break
for name, val in rows:
if type(val).__name__ in _ABSENT:
continue
found.append((int(str(name).rsplit(".", 1)[-1]), str(val)))
if len(found) >= INTERFACES_LISTED:
break
if found:
out["interfaces"] = found
break
return out
except Exception as exc:
logger.warning("SNMP test of %s:%s failed: %s", address, port, exc)
return {"error": f"SNMP test failed: {exc}"}
return results
finally:
_close_engine(engine)

Some files were not shown because too many files have changed in this diff Show More