Merge pull request 'Ansible schedule form: structured playbook-variable fields + first-item dropdown defaults' (#2) from dev into main
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 46s
CI / integration (push) Successful in 2m21s
CI / publish (push) Successful in 4s

This commit was merged in pull request #2.
This commit is contained in:
2026-06-30 23:44:04 -04:00
290 changed files with 26118 additions and 1482 deletions
+3 -2
View File
@@ -26,8 +26,9 @@ config.yaml
# Runtime data (mounted at runtime via volume) # Runtime data (mounted at runtime via volume)
playbook_cache/ playbook_cache/
# Plugins (mounted at runtime via volume) # NOTE: first-party plugins under plugins/ now ship IN the image
plugins/ # (Dockerfile `COPY plugins/`), so plugins/ must NOT be ignored here.
# Third-party plugins are mounted at runtime into /data/plugins instead.
# Deployment files not needed in image # Deployment files not needed in image
docker-compose.yml docker-compose.yml
+7 -2
View File
@@ -1,6 +1,11 @@
# Copy to .env for Docker / local development secrets # Copy to .env for Docker / local development secrets
# These override values in config.yaml # These override values in config.yaml
STEWARD_SECRET_KEY=change-me STEWARD_DATABASE_URL=postgresql+asyncpg://steward:password@localhost/steward
STEWARD_DATABASE__URL=postgresql+asyncpg://steward:password@localhost/steward
STEWARD_SMTP__PASSWORD= STEWARD_SMTP__PASSWORD=
# --- compose.deploy.yml (remote instance) only ---
# Password for the bundled Postgres; the steward service builds its
# STEWARD_DATABASE_URL from it. Leave STEWARD_SECRET_KEY above unset to let the
# app generate + persist one on the /data volume.
POSTGRES_PASSWORD=change-me
+129
View File
@@ -0,0 +1,129 @@
name: CI
# CI lanes per FabledRulebook forgejo.md "CI philosophy":
# - lint: ruff only, no dep install — fast-fail for the common lint bounce.
# - unit: pytest -m "not integration"; no service containers, no DB.
# - integration: postgres service container; boots the real app + runs all
# (core + bundled-plugin) migrations.
# Single repo: first-party plugins are bundled in-tree under plugins/, so the
# unit lane imports them directly — no cross-repo checkout.
on:
push:
branches: [dev, main]
# pull_request intentionally absent — push on [dev, main] already fires CI for
# every dev commit and dev→main PRs. Single-operator repo, no fork PRs.
jobs:
# Fast-fail lint lane. ruff is pre-installed in the ci-python image, so this
# runs with NO dependency install and surfaces lint bounces in seconds.
lint:
runs-on: python-ci
container:
image: git.fabledsword.com/bvandeusen/ci-python:3.14
steps:
- uses: actions/checkout@v4
- name: Ruff lint
run: ruff check steward/ plugins/ tests/
unit:
runs-on: python-ci
container:
image: git.fabledsword.com/bvandeusen/ci-python:3.14
steps:
- uses: actions/checkout@v4
- name: Install Steward + test deps
# uv: 5-10x faster wheel resolve than pip on cold caches. Falls back to
# pip on uv-missing runners. Steward declares its deps in pyproject; the
# [dev] extra adds pytest + pytest-asyncio.
run: |
if command -v uv >/dev/null 2>&1; then
uv pip install --system -e '.[dev]'
else
pip install -e '.[dev]'
fi
- name: Pytest (unit only — integration runs in its own lane)
run: pytest tests/ -v -m "not integration"
integration:
runs-on: python-ci
container:
image: git.fabledsword.com/bvandeusen/ci-python:3.14
env:
STEWARD_SECRET_KEY: ci_integration_placeholder
services:
postgres:
image: postgres:16-alpine
env:
POSTGRES_USER: steward
POSTGRES_PASSWORD: steward
POSTGRES_DB: steward
options: >-
--health-cmd "pg_isready -U steward"
--health-interval 10s
--health-timeout 5s
--health-retries 10
steps:
- uses: actions/checkout@v4
- name: Boot + migrate against Postgres
# act_runner (swarm-runner) puts service containers on the default bridge
# with no embedded DNS, so the hostname `postgres` won't resolve — we
# discover the service container's bridge IP via the docker socket. The
# job name `integration` (no underscore) is the docker-ps name filter.
run: |
set -euxo pipefail
echo "=== container landscape (diagnostic for filter scoping) ==="
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
echo "=== end landscape ==="
PG=$(docker ps --filter "name=integration" --filter "ancestor=postgres:16-alpine" -q | head -n1)
test -n "$PG"
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
test -n "$PG_IP"
export STEWARD_DATABASE_URL="postgresql+asyncpg://steward:steward@$PG_IP:5432/steward"
for i in $(seq 1 60); do
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
sleep 2
done
if command -v uv >/dev/null 2>&1; then
uv pip install --system -e '.[dev,ansible]'
else
pip install -e '.[dev,ansible]'
fi
pytest tests/integration -v -m integration --durations=10
# Image-publish lane — FabledRulebook rule 46: every push to dev/main publishes
# an immutable commit-SHA image (the rollback unit); dev moves :dev, main moves
# :latest. Gated on the three test lanes via `needs:` so a red commit never
# produces a :dev image. Mechanics mirror CI-runner's build-ci-python.yml.
publish:
needs: [lint, unit, integration]
runs-on: python-ci
container:
# ci-builder carries docker + buildx + git. The docker socket is
# auto-mounted by act_runner (valid_volumes) — do NOT add a
# container.options "-v /var/run/docker.sock:..." mount; that duplicate
# mount fails the job at "Set up job".
image: git.fabledsword.com/bvandeusen/ci-builder:latest
steps:
- uses: actions/checkout@v4
- name: Registry login
# The injected GITHUB_TOKEN lacks write:package scope, so docker push
# 401s with it. REGISTRY_TOKEN is a repo Actions secret holding a
# Forgejo token scoped read:package + write:package.
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.fabledsword.com -u bvandeusen --password-stdin
- name: Build + push (commit-SHA always; :dev on dev, :latest on main)
run: |
set -euxo pipefail
IMAGE=git.fabledsword.com/bvandeusen/steward
docker build -t "$IMAGE:${{ github.sha }}" .
docker push "$IMAGE:${{ github.sha }}"
if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:dev"
docker push "$IMAGE:dev"
elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
docker tag "$IMAGE:${{ github.sha }}" "$IMAGE:latest"
docker push "$IMAGE:latest"
fi
- name: Prune dangling layers
if: always()
run: docker image prune -f
+10 -3
View File
@@ -1,9 +1,10 @@
# Planning files # Planning files
docs/superpowers/ docs/superpowers/
# Plugin directory — managed as a separate repo (bvandeusen/steward-plugins) # First-party plugins are tracked in-tree under plugins/ and ship in the image.
# PLUGIN_DIR in config.yaml points here at runtime # Third-party plugins are mounted at runtime into the external dir
/plugins/ # (STEWARD_PLUGIN_DIR, default /data/plugins) and are not tracked here.
/plugins/**/*.zip
# Python # Python
__pycache__/ __pycache__/
@@ -50,3 +51,9 @@ Thumbs.db
# Playbook cache (runtime data) # Playbook cache (runtime data)
playbook_cache/ playbook_cache/
# Superpowers brainstorm scratch (visual companion mockups, state)
.superpowers/
# Local dev Ansible playbooks (operator content, mounted into the container)
/ansible-dev/
+13 -1
View File
@@ -5,6 +5,11 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \
iputils-ping \ iputils-ping \
# gosu — minimal privilege-drop tool used by entrypoint.sh # gosu — minimal privilege-drop tool used by entrypoint.sh
gosu \ gosu \
# openssh-client — ansible-playbook reaches remote hosts over ssh
openssh-client \
# sshpass — Ansible shells out to it for first-contact password SSH
# (provisioning a fresh host before the managed key is installed)
sshpass \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Upgrade pip to suppress the version notice # Upgrade pip to suppress the version notice
@@ -14,9 +19,16 @@ WORKDIR /app
COPY pyproject.toml . COPY pyproject.toml .
COPY steward/ steward/ COPY steward/ steward/
RUN pip install --no-cache-dir . # Bundle the extras whose first-party plugins ship in the image: [ansible] for
# the playbook runner, [snmp] (pysnmp) for the SNMP poller. Without them those
# bundled plugins load but silently no-op for want of their runtime dependency.
RUN pip install --no-cache-dir '.[ansible,snmp]'
COPY alembic.ini . COPY alembic.ini .
# First-party plugins ship inside the image (bundled root at /app/plugins).
# Third-party plugins are mounted at runtime into the external root
# (STEWARD_PLUGIN_DIR, default /data/plugins).
COPY plugins/ plugins/
COPY entrypoint.sh /entrypoint.sh COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh RUN chmod +x /entrypoint.sh
+36
View File
@@ -28,6 +28,42 @@ docker compose up -d
Open `http://localhost:5000`. On first run you'll be prompted to create the admin account. Open `http://localhost:5000`. On first run you'll be prompted to create the admin account.
This compose file builds the image locally and bind-mounts the source for live editing. To run a persistent instance off the published CI image instead, see **Remote dev instance** below.
---
## Remote dev instance
For a long-lived instance on a remote server that tracks the `dev` CI line, use `compose.deploy.yml`. It pulls the published image (`git.fabledsword.com/bvandeusen/steward:dev`) rather than building, runs no source bind-mounts, and persists data in named volumes.
**One-time prerequisite — registry push token (on the Git host).** CI publishes the image, so the FabledSteward repo needs a push credential. The injected `GITHUB_TOKEN` lacks `write:package`, so create a Forgejo token scoped **`read:package` + `write:package`** and add it as the repo Actions secret **`REGISTRY_TOKEN`**. Until this exists, the `publish` CI job 401s and no `:dev` image is produced.
**Standup (on the server):**
```bash
# 1. Authenticate to the registry (read:package token is enough to pull)
docker login git.fabledsword.com -u <user>
# 2. Configure secrets
cp .env.example .env
# Set POSTGRES_PASSWORD. Leave STEWARD_SECRET_KEY unset to let the app
# generate + persist one on the /data volume.
# 3. Pull + boot
docker compose -f compose.deploy.yml up -d
```
Open `http://<server>:5000` and create the admin account on first run.
**Update to the latest dev build:**
```bash
docker compose -f compose.deploy.yml pull
docker compose -f compose.deploy.yml up -d
```
Every push to `dev` that goes CI-green publishes a fresh `:dev` (and an immutable `:<commit-sha>` for rollback). To pin a specific commit, change the `image:` tag in `compose.deploy.yml` to `:<sha>`.
--- ---
## Quick Start — Bare Metal ## Quick Start — Bare Metal
+58
View File
@@ -0,0 +1,58 @@
# CI Requirements — Steward
> Spec: https://git.fabledsword.com/bvandeusen/CI-runner/src/branch/main/docs/process.md
## Runtime image
git.fabledsword.com/bvandeusen/ci-python:3.14
## Image deps used
- python 3.14
- ruff (analyzer for `steward/`, `plugins/`, `tests/`)
- docker CLI (integration lane: bridge-IP discovery of the Postgres service container)
## Per-job tool installs
- `uv pip install --system -e '.[dev]'` (pip fallback) — in the `unit` job.
- `uv pip install --system -e '.[dev,ansible]'` (pip fallback) — in the
`integration` job, which boots the real app and exercises the Ansible runner,
so it needs the `ansible` extra (`ansible>=10,<13`) at import time.
Steward declares its deps in `pyproject.toml`; the `[dev]` extra adds
`pytest` + `pytest-asyncio`. No `requirements.txt` is tracked.
## Lanes
- **lint** — `ruff check steward/ plugins/ tests/`, no dep install.
- **unit** — `pytest -m "not integration"`. The whole current suite runs here:
it builds the app with `testing=True`, which mocks `db_sessionmaker`, so no DB
is touched. First-party plugins are bundled in-tree under `plugins/`, so plugin
unit tests import them directly — no cross-repo checkout.
- **integration** — `pytest tests/integration -m integration` against a
`postgres:16-alpine` service. The single test boots the real app
(`create_app(testing=False)`), which runs core + every bundled plugin's
migrations, then round-trips the DB. This is the guard that the folded-in
plugin migration graph resolves.
- **publish** — builds the runtime `Dockerfile` and pushes to the package
registry. Runs in `ci-builder:latest` (docker + buildx, socket auto-mounted),
gated on `needs: [lint, unit, integration]` so a red commit never ships an
image. Publishes `git.fabledsword.com/bvandeusen/steward:<sha>` always, plus
`:dev` on `dev` and `:latest` on `main` (FabledRulebook rule 46). Needs the
`REGISTRY_TOKEN` repo Actions secret (`read:package` + `write:package`) —
the injected `GITHUB_TOKEN` can't push packages.
## Notes
- Steward has **no frontend and no Redis/Celery** (no external workers), so there
is no frontend-build lane and the only service container is Postgres.
- Integration uses Forgejo Actions `services:` + a socket-discovered bridge IP
because `act_runner` (swarm-runner v0.6+) puts services on the default bridge
with no embedded DNS. FabledCurator's `ci.yml` is the canonical example of the
pattern; Steward's is the same shape minus Redis and minus sharding.
- The job name `integration` has no underscore — `act_runner` strips underscores
from job names when building service-container labels, so the `docker ps`
name filter uses the bare job name.
- Migrations run via the app itself (`create_app``run_core_migrations`, async
through `asyncpg`), so the integration lane needs no separate `alembic upgrade`
step and no psycopg/sync driver.
+69
View File
@@ -0,0 +1,69 @@
# Remote deployment — pulls the published :dev image instead of building locally.
#
# This is NOT the local-dev compose. `docker-compose.yml` bind-mounts ./steward
# + ./plugins and runs --debug for live editing; THIS file runs the image as the
# source of truth (no bind-mounts, no reloader) for a persistent remote instance
# that tracks the dev CI line.
#
# Standup + update runbook: see README → "Remote dev instance".
services:
steward:
image: git.fabledsword.com/bvandeusen/steward:dev
container_name: steward
# Re-pull :dev on every `up` so restarting the stack picks up the latest
# green dev build without a manual `docker pull`.
pull_policy: always
ports:
- "5000:5000"
networks:
- backend
volumes:
# /data holds the auto-generated secret key, third-party plugins, and the
# Ansible playbook cache — all of which must survive image updates. No
# source bind-mounts: core + first-party plugins live inside the image.
#
# The container runs as the unprivileged 'app' user (uid 1000), so /data
# MUST be writable by uid 1000. A named docker volume (below) gets that
# automatically. If you bind-mount a host/NFS path instead, chown it to
# 1000:1000 (and mind NFS root_squash) — otherwise the app can't persist
# its secret key, mints a fresh one each boot, and every encrypted secret
# (managed SSH key, SMTP/OIDC/LDAP creds) becomes unrecoverable. The app
# now refuses to start in that state rather than silently degrade.
- app_data:/data
environment:
- STEWARD_DATABASE_URL=postgresql+asyncpg://steward:${POSTGRES_PASSWORD:-steward}@db/steward
# STEWARD_SECRET_KEY is intentionally absent — the app generates a random
# key on first boot and persists it to /data/secret.key, which the app_data
# volume keeps across image updates. On multi-node Swarm with a shared bind
# mount, pin STEWARD_SECRET_KEY (env or a docker secret) instead, so the key
# never depends on filesystem ownership being correct on every node.
depends_on:
db:
condition: service_healthy
restart: unless-stopped
db:
image: postgres:16-alpine
networks:
- backend
environment:
POSTGRES_USER: steward
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-steward}
POSTGRES_DB: steward
volumes:
- pgdata:/var/lib/postgresql/data
# No host port published — db is reachable only on the backend network.
healthcheck:
test: ["CMD-SHELL", "pg_isready -U steward"]
interval: 5s
timeout: 5s
retries: 5
restart: unless-stopped
networks:
backend:
volumes:
pgdata:
app_data:
+20 -3
View File
@@ -3,15 +3,29 @@ services:
build: . build: .
container_name: steward container_name: steward
image: steward:latest image: steward:latest
# Dev: --debug turns on Quart's auto-reloader so edits to the mounted source
# below take effect without a rebuild.
command: ["steward", "--host", "0.0.0.0", "--port", "5000", "--debug"]
ports: ports:
- "5000:5000" - "5000:5000"
volumes: volumes:
- app_data:/data - app_data:/data
- ./plugins:/app/plugins:ro # Live-edit core + first-party plugins without rebuilding. PYTHONPATH=/app
- /mnt/Data/traefik/log:/var/log/traefik:ro # (below) makes this mounted source win over the image-installed package.
- /var/run/docker.sock:/var/run/docker.sock:ro - ./steward:/app/steward
- ./plugins:/app/plugins
# Dev playbooks: edit on the host under ./ansible-dev/, register a *local*
# Ansible source pointing at /srv/ansible in Settings → Ansible.
- ./ansible-dev:/srv/ansible
# Optional host mounts — enable the matching plugin in Settings first, then
# uncomment (the traefik path must exist on this host):
# - /mnt/Data/traefik/log:/var/log/traefik:ro # traefik plugin
# - /var/run/docker.sock:/var/run/docker.sock:ro # docker plugin
environment: environment:
- STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@db/steward - STEWARD_DATABASE_URL=postgresql+asyncpg://steward:steward@db/steward
# Dev-only fixed key so sessions survive restarts. Replace in production.
- STEWARD_SECRET_KEY=dev-secret-key-change-me
- PYTHONPATH=/app
depends_on: depends_on:
db: db:
condition: service_healthy condition: service_healthy
@@ -23,6 +37,9 @@ services:
POSTGRES_USER: steward POSTGRES_USER: steward
POSTGRES_PASSWORD: steward POSTGRES_PASSWORD: steward
POSTGRES_DB: steward POSTGRES_DB: steward
ports:
# Exposed for dev convenience (psql from the host). Drop for prod.
- "5432:5432"
volumes: volumes:
- pgdata:/var/lib/postgresql/data - pgdata:/var/lib/postgresql/data
healthcheck: healthcheck:
+191
View File
@@ -0,0 +1,191 @@
# Authoring Steward-friendly Ansible playbooks
This is the contract between a playbook and Steward's run UI. Follow it and a
playbook drops into Steward with a description, fill-in variable fields, correct
targeting, and credentials supplied automatically — no per-playbook wiring.
It applies to **any** playbook in a configured source (bundled, the writable
local source, or a git source), including third-party ones.
---
## 1. Describe what it does — `# description:`
Steward shows a one-line description when a playbook is selected in the run form.
```yaml
---
# description: Reclaim disk on Docker/Swarm nodes by pruning unused images and build cache.
- name: Docker system prune
hosts: all
...
```
- Format: a comment line `# description: <text>` anywhere in the file. **First
match wins.** Case-insensitive on the `description:` key.
- Why a comment and not a key: Ansible rejects unknown *play* keys (you can't add
`description:` to a play), so a comment is the portable place. It survives
`ansible-playbook` untouched.
- Fallback: if there's no `# description:` comment, Steward uses the **first
play's `name:`**. So always give your play a meaningful `name:` even without
the comment.
- Keep it to one readable line. Longer "how to use" notes can go in additional
normal comments — Steward only reads the `description:` line.
## 2. Declare tunables in `vars:` — they become fill-in fields
Every **scalar** entry in a play's `vars:` block becomes an editable field in the
run form, with the default shown as the input's placeholder.
```yaml
vars:
prune_all_images: false # → checkbox-ish text field, placeholder "default: false"
keep_last_days: 7 # → field, placeholder "default: 7"
registry_url: "" # → field, placeholder "no default"
```
- **Blank field = use the default.** Steward only sends fields the operator
actually fills, so an untouched field falls through to the playbook/inventory
default rather than overriding it.
- Only scalars (string/int/float/bool) surface as fields. Lists/dicts are
skipped — set those in the playbook or via inventory group/host vars.
- Values are delivered as **extra-vars** (`-e`), which are the **highest**
precedence in Ansible — they override the `vars:` defaults. (This is why the
default can be empty and still be safely overridden at run time.)
### `vars_prompt:` also works
Steward reads `vars_prompt` too. Use it when you want an explicit prompt or a
required value:
```yaml
vars_prompt:
- name: release_tag
prompt: "Which release to deploy?" # shown as the field label
# no default → Steward marks the field REQUIRED
- name: admin_password
prompt: "Admin password"
private: true # → masked field, never stored
```
## 3. Secrets — name them so they're masked and not persisted
A field is treated as **secret** (rendered masked, and its value is **never
written to the DB / run history**) when either:
- the variable name contains `password`, `passwd`, `secret`, `token`,
`api_key` / `apikey`, `private_key`, or `credential` (case-insensitive), **or**
- it's a `vars_prompt` entry with `private: true` (Ansible's default for
vars_prompt is private).
So name sensitive variables accordingly (`db_password`, `api_token`,
`vault_secret`) and they're handled safely with no extra config. Non-secret
run-time vars are persisted (so scheduled runs can reuse them); secret ones are
passed to the run only.
## 4. Target with `hosts: all`
Steward builds the inventory itself from the **target / group** the operator
picks in the run form (or the single host on a host page). Your play should:
```yaml
hosts: all # run against whatever Steward scoped to
```
- Don't hardcode hostnames or rely on a checked-in inventory for Steward runs
(Steward generates a fresh inventory per run).
- Per-host connection vars (`ansible_host`, plus anything you set on the target
in **Ansible → Inventory**) arrive as inventory host vars.
- The run form's **Limit** / **Tags** map to `--limit` / `--tags`.
## 5. Privileges & connection — don't put credentials in the playbook
Steward supplies SSH and become for you:
- Steady-state runs connect as the managed **`steward`** account using Steward's
managed key; that account has **passwordless sudo**. So just use
`become: true` where you need root.
- First-contact provisioning uses a one-time bootstrap user/password the
operator enters (never stored).
- Never embed SSH keys, passwords, or `ansible_user`/`ansible_ssh_pass` in the
playbook. Connection identity is global (Settings → Ansible) or per-target.
## 6. Be idempotent
Steward re-runs playbooks (updates, schedules, retries). Use modules that
converge (state-based) rather than ad-hoc `command:`/`shell:` where possible, so
re-runs are safe.
---
## What Steward reads from a playbook (summary)
| Source in the playbook | What Steward does with it |
|---|---|
| `# description: <text>` comment | Description shown on selection (first match) |
| first play `name:` | Description fallback |
| `# steward:category: <text>` | Grouping badge in the browse list + run form |
| `# steward:confirm: true` | Requires a confirmation tick before the run launches |
| `vars:` scalar entries | Run-time fill-in fields (placeholder = default) |
| `vars_prompt:` entries | Run-time fields (required if no default) |
| secret-looking var name / `private: true` | Field masked + value not persisted |
| `hosts:` | Expected to be `all`; Steward provides the inventory |
Everything else (SSH user/key, become password, the inventory, `steward_token`
etc. for the agent playbooks) is injected by Steward at run time.
## Metadata comments
Steward reads metadata from magic comments (Ansible rejects unknown play keys,
so comments are the portable place). Two forms:
- `# description: <text>` — the description (see §1).
- `# steward:<key>: <value>` — the namespaced metadata block. First match per
key wins; keys are case-insensitive.
**Implemented `# steward:` keys:**
| Key | Example | Effect |
|---|---|---|
| `category` | `# steward:category: maintenance` | Free-text grouping label. Shown as a badge in the browse list and on the run form. Purely organizational. |
| `confirm` | `# steward:confirm: true` | Marks the playbook as significant/destructive. The run form then requires an explicit confirmation tick before it can be launched. Use for data-loss-capable plays (prune with volumes, resets, etc.). Truthy values: `true`/`yes`/`1`/`on`. |
| `description` | `# steward:description: ...` | Alias for `# description:` (the unprefixed form takes precedence if both exist). |
```yaml
---
# description: Reclaim disk on Docker/Swarm nodes by pruning unused data.
# steward:category: maintenance
# steward:confirm: true
- name: Docker system prune
hosts: all
...
```
Other `# steward:<key>:` keys are simply ignored today — the namespace is
reserved, so it's safe to add ones Steward doesn't yet understand without
breaking anything, but only `category`, `confirm`, and `description` do something.
## Minimal annotated example
```yaml
---
# description: Restart a systemd service and confirm it came back up.
- name: Restart a service
hosts: all
become: true
gather_facts: false
vars:
service_name: "" # required-ish: operator fills it in the run form
tasks:
- name: Validate input
ansible.builtin.assert:
that: service_name | default('') | length > 0
fail_msg: "Set service_name."
- name: Restart
ansible.builtin.systemd:
name: "{{ service_name }}"
state: restarted
- name: Confirm active
ansible.builtin.command: "systemctl is-active {{ service_name }}"
changed_when: false
```
File diff suppressed because it is too large Load Diff
+1
View File
@@ -0,0 +1 @@
# plugins/__init__.py
+54
View File
@@ -0,0 +1,54 @@
# plugins/docker/__init__.py
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from quart import Quart
_app: "Quart | None" = None
def setup(app: "Quart") -> None:
global _app
_app = app
from .models import DockerContainer, DockerMetric # noqa: F401 — register with Base
# Publish the per-host persist hook so the host agent can hand off the
# `docker` array from each sample without importing our models (opportunistic
# synergy — host_agent gates on has_capability and no-ops if we're disabled).
# required_role=viewer: this is a trusted server-side data-plane write driven
# by the authenticated agent ingest, not a user-facing privileged action.
from steward.core.capabilities import register_capability
from steward.models.users import UserRole
from .ingest import persist_host_docker
from .retention import run_docker_retention
register_capability(
"docker.persist_host_samples", persist_host_docker,
label="Persist host Docker samples",
description="Store per-host container state + metrics pushed by the host agent.",
required_role=UserRole.viewer,
)
# Roll up + prune Docker time-series, driven by the core cleanup task without
# it importing our models. Same trusted server-side data-plane role as above.
register_capability(
"docker.run_retention", run_docker_retention,
label="Run Docker retention",
description="Roll up old docker_metrics to hourly + prune stale metrics/events.",
required_role=UserRole.viewer,
)
def get_scheduled_tasks() -> list:
# Collection is now agent-driven (pushed to the host_agent ingest); the
# central socket scrape was removed. No periodic task to register.
return []
def get_blueprint():
from .routes import docker_bp
return docker_bp
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (fleet-wide container view).
return [{"label": "Docker", "href": "/plugins/docker/"}]
+27
View File
@@ -0,0 +1,27 @@
"""Model-free Docker view helpers.
Kept separate from routes.py/models.py so tests can import it directly: importing
a module that re-imports plugins.docker.models AFTER the app has loaded the docker
plugin raises "Table 'docker_containers' is already defined". This module touches
no ORM models, so it's safe to import in either environment.
"""
from __future__ import annotations
def dedup_by_container_id(containers: list) -> list:
"""Collapse containers double-reported across hosts. Swarm-aware agents on
every manager list cluster-wide tasks, so the same container (identical
container_id) arrives once per manager and would otherwise be counted N times.
Keep the first occurrence per non-empty container_id — callers order
running-first, so a running report wins over a stale stopped one. Rows without
a container_id (older agents) can't be matched, so they're kept as-is."""
seen: set[str] = set()
out = []
for c in containers:
cid = (c.container_id or "").strip()
if cid:
if cid in seen:
continue
seen.add(cid)
out.append(c)
return out
+336
View File
@@ -0,0 +1,336 @@
# plugins/docker/ingest.py
"""Persist host-scoped Docker samples pushed by the host agent.
Published as the "docker.persist_host_samples" capability (see __init__.setup),
so the host_agent plugin can hand off a sample's `docker` array (and, on a swarm
manager, its `swarm` object) WITHOUT importing the docker models — the coupling
is opportunistic and degrades to a no-op when the docker plugin is disabled.
Runs inside the caller's open transaction (the ingest handler's SAVEPOINT);
never opens or commits its own.
"""
from __future__ import annotations
import json
from datetime import datetime
from sqlalchemy import delete, select
# Stopped/terminal container states (anything not "running"/"paused"/"restarting").
_STOPPED_STATES = {"exited", "dead", "stopped"}
def _parse_started_at(value) -> datetime | None:
"""Parse the agent's ISO-8601 started_at string back to a datetime."""
if not isinstance(value, str) or not value:
return None
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
def _derive_events(old_state: dict, new_containers: list) -> list:
"""Diff a fresh snapshot against stored per-container state → lifecycle events.
Pure function (no DB) so it's unit-testable. `old_state` maps name → the
previously stored {status, health, oom_killed, exit_code}; `new_containers`
is the newest snapshot's list of container dicts. Returns a list of
(container_name, event, detail) tuples:
start — a container transitions into running (or a genuinely new
container appears already running)
stop — running → not-running, or a running container is removed
die — same as stop but with a non-zero exit code (abnormal)
oom — OOMKilled flips False→True
health_change — HEALTHCHECK status string changes
The CALLER skips this entirely on a host's first-ever snapshot (empty
old_state) so the baseline doesn't emit a spurious "start" per existing
container — a later-appearing container still gets one because by then
old_state is populated and that container's prior entry is simply absent.
"""
events: list = []
new_by_name = {c["name"]: c for c in new_containers if c.get("name")}
for name, c in new_by_name.items():
new_status = (c.get("status") or "").lower()
new_running = new_status == "running"
new_oom = bool(c.get("oom_killed"))
new_health = c.get("health")
new_exit = c.get("exit_code")
old = old_state.get(name)
if old is None:
# Newly observed container — only a start is meaningful (we have no
# prior state to diff a death against).
if new_running:
events.append((name, "start", c.get("image") or None))
continue
old_running = (old.get("status") or "").lower() == "running"
if new_running and not old_running:
events.append((name, "start", c.get("image") or None))
elif old_running and not new_running:
if new_exit not in (None, 0):
events.append((name, "die", f"exit {new_exit}"))
else:
events.append((name, "stop", None))
if new_oom and not bool(old.get("oom_killed")):
events.append((name, "oom", None))
if new_health != old.get("health") and (new_health or old.get("health")):
events.append((name, "health_change",
f"{old.get('health') or '?'}{new_health or '?'}"))
# A running container that vanished from the listing entirely (removed).
for name, old in old_state.items():
if name not in new_by_name and (old.get("status") or "").lower() == "running":
events.append((name, "stop", "removed"))
return events
async def _persist_swarm(session, host, swarm: dict) -> None:
"""Upsert this manager's swarm topology; drop rows no longer reported.
Current-state tables (not time-series): a manager re-reports its full
services/nodes set every sample, so we upsert what's present and delete what
isn't (scoped to this host so two managers don't clobber each other).
"""
from datetime import timezone
from .models import DockerSwarmNode, DockerSwarmService
now = datetime.now(timezone.utc)
services = swarm.get("services") or []
seen_services: set[str] = set()
for s in services:
name = s.get("service_name")
if not name:
continue
seen_services.add(name)
row = await session.get(DockerSwarmService, (host.id, name))
if row is None:
row = DockerSwarmService(host_id=host.id, service_name=name)
session.add(row)
row.mode = s.get("mode") or "replicated"
row.desired = int(s.get("desired") or 0)
row.running = int(s.get("running") or 0)
row.image = s.get("image")
row.placement_json = json.dumps(s.get("placement") or [])
row.updated_at = now
stale_services = delete(DockerSwarmService).where(
DockerSwarmService.host_id == host.id)
if seen_services:
stale_services = stale_services.where(
DockerSwarmService.service_name.notin_(seen_services))
await session.execute(stale_services)
nodes = swarm.get("nodes") or []
seen_nodes: set[str] = set()
for n in nodes:
nid = n.get("node_id")
if not nid:
continue
seen_nodes.add(nid)
row = await session.get(DockerSwarmNode, (host.id, nid))
if row is None:
row = DockerSwarmNode(host_id=host.id, node_id=nid)
session.add(row)
row.hostname = n.get("hostname") or ""
row.role = n.get("role") or "worker"
row.availability = n.get("availability") or "active"
row.status = n.get("status") or "unknown"
row.leader = bool(n.get("leader", False))
row.updated_at = now
stale_nodes = delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id)
if seen_nodes:
stale_nodes = stale_nodes.where(DockerSwarmNode.node_id.notin_(seen_nodes))
await session.execute(stale_nodes)
async def _persist_disk(session, host, disk: dict) -> None:
"""Upsert this host's /system/df summary + replace its image storage rows.
Current-state (not time-series): the agent re-reports the full summary each
interval, so we overwrite the single summary row and replace the image set
(delete rows no longer present, upsert the rest), scoped to this host.
"""
from datetime import timezone
from .models import DockerDiskUsage, DockerImage
now = datetime.now(timezone.utc)
summary = await session.get(DockerDiskUsage, host.id)
if summary is None:
summary = DockerDiskUsage(host_id=host.id)
session.add(summary)
summary.layers_size = int(disk.get("layers_size") or 0)
summary.images_size = int(disk.get("images_size") or 0)
summary.images_reclaimable = int(disk.get("images_reclaimable") or 0)
summary.containers_size = int(disk.get("containers_size") or 0)
summary.volumes_size = int(disk.get("volumes_size") or 0)
summary.build_cache_size = int(disk.get("build_cache_size") or 0)
summary.images_total = int(disk.get("images_total") or 0)
summary.images_active = int(disk.get("images_active") or 0)
summary.containers_count = int(disk.get("containers_count") or 0)
summary.volumes_count = int(disk.get("volumes_count") or 0)
summary.updated_at = now
images = disk.get("images") or []
seen: set[str] = set()
for im in images:
iid = im.get("image_id")
if not iid or iid in seen:
continue
seen.add(iid)
row = await session.get(DockerImage, (host.id, iid))
if row is None:
row = DockerImage(host_id=host.id, image_id=iid)
session.add(row)
row.repo_tag = (im.get("repo_tag") or "<none>")[:512]
row.size = int(im.get("size") or 0)
row.shared_size = int(im.get("shared_size") or 0)
row.containers = int(im.get("containers") or 0)
row.updated_at = now
stale = delete(DockerImage).where(DockerImage.host_id == host.id)
if seen:
stale = stale.where(DockerImage.image_id.notin_(seen))
await session.execute(stale)
async def persist_host_docker(session, host, snapshots, swarm=None, disk=None) -> None:
"""Upsert containers + time-series + lifecycle events + swarm for one host.
`snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
one entry per ingested sample carrying docker data (usually one; more when
the agent flushes a backlog). Every snapshot contributes time-series
DockerMetric rows; the newest snapshot drives current container state, the
alert pipeline, and lifecycle-event derivation. `swarm` is the newest
sample's swarm object (or None off managers) — persisted when present.
`disk` is the newest sample's /system/df summary (or None on Docker-less
hosts) — persisted when present.
"""
from steward.core.alerts import record_metric
from .models import DockerContainer, DockerEvent, DockerMetric
if swarm is not None:
await _persist_swarm(session, host, swarm)
if disk is not None:
await _persist_disk(session, host, disk)
if not snapshots:
return
ordered = sorted(snapshots, key=lambda s: s[0])
latest_at, latest_containers = ordered[-1]
# Time-series points for every snapshot (running containers with a CPU read).
for recorded_at, containers in ordered:
for c in containers:
if c.get("status") == "running" and c.get("cpu_pct") is not None:
session.add(DockerMetric(
host_id=host.id,
container_name=c["name"],
scraped_at=recorded_at,
cpu_pct=c["cpu_pct"],
mem_pct=c.get("mem_pct") or 0.0,
mem_usage_bytes=c.get("mem_usage_bytes") or 0,
))
# Snapshot of stored per-container state BEFORE the upsert overwrites it —
# used to diff lifecycle events. Skip event derivation on the host's first
# snapshot (empty state) so we don't emit a start per pre-existing container.
old_rows = (await session.execute(
select(DockerContainer).where(DockerContainer.host_id == host.id)
)).scalars().all()
old_state = {
r.name: {"status": r.status, "health": r.health,
"oom_killed": r.oom_killed, "exit_code": r.exit_code}
for r in old_rows
}
if old_state:
for name, event, detail in _derive_events(old_state, latest_containers):
session.add(DockerEvent(
host_id=host.id, container_name=name,
event=event, detail=detail, at=latest_at,
))
# Current state + alerts from the newest snapshot only.
for c in latest_containers:
name = c.get("name")
if not name:
continue
existing = await session.get(DockerContainer, (host.id, name))
if existing is None:
existing = DockerContainer(host_id=host.id, name=name)
session.add(existing)
existing.container_id = c.get("container_id", "") or ""
existing.image = c.get("image", "") or ""
existing.status = c.get("status", "unknown") or "unknown"
existing.cpu_pct = c.get("cpu_pct")
existing.mem_usage_bytes = c.get("mem_usage_bytes")
existing.mem_limit_bytes = c.get("mem_limit_bytes")
existing.mem_pct = c.get("mem_pct")
existing.ports_json = json.dumps(c.get("ports") or [])
existing.started_at = _parse_started_at(c.get("started_at"))
existing.scraped_at = latest_at
# Enrichment (agent ≥ 1.4.0; .get keeps older agents working — fields
# stay None/0 when absent from the payload).
existing.restart_count = c.get("restart_count", 0) or 0
existing.health = c.get("health")
existing.exit_code = c.get("exit_code")
existing.oom_killed = bool(c.get("oom_killed", False))
existing.compose_project = c.get("compose_project")
existing.service_name = c.get("service_name")
existing.task_id = c.get("task_id")
existing.node_id = c.get("node_id")
existing.net_rx_bytes = c.get("net_rx_bytes")
existing.net_tx_bytes = c.get("net_tx_bytes")
existing.blk_read_bytes = c.get("blk_read_bytes")
existing.blk_write_bytes = c.get("blk_write_bytes")
# Alert pipeline — resource is host-scoped so containers of the same name
# on different hosts don't collide in the metric/alert namespace.
resource = f"{host.name}/{name}"
if c.get("status") == "running" and c.get("cpu_pct") is not None:
await record_metric(
session=session, source_module="docker",
resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
)
if c.get("mem_pct") is not None:
await record_metric(
session=session, source_module="docker",
resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
)
# Restart count is alertable regardless of state (crash-looping matters
# most while the container is down/restarting).
await record_metric(
session=session, source_module="docker",
resource_name=resource, metric_name="restart_count",
value=float(c.get("restart_count", 0) or 0),
)
# Health → 1.0/0.0 only for containers that actually define a HEALTHCHECK
# (health is None otherwise — recording 0 would false-alarm every plain
# container).
health = c.get("health")
if health in ("healthy", "unhealthy", "starting"):
await record_metric(
session=session, source_module="docker",
resource_name=resource, metric_name="is_healthy",
value=1.0 if health == "healthy" else 0.0,
)
# Reap containers that vanished from the host's latest listing. Without this,
# removed one-shot containers (CI job runners, buildkit builders, codex jobs)
# accumulate forever as permanent "stopped" rows and inflate every count.
# The listing is authoritative (event derivation already treats absence as
# removal), so anything not in the newest snapshot no longer exists on the
# host. Mirrors the image/swarm/disk persisters. An empty snapshot legitimately
# means the host has no containers, so the unfiltered delete is correct.
seen_names = {c["name"] for c in latest_containers if c.get("name")}
reap = delete(DockerContainer).where(DockerContainer.host_id == host.id)
if seen_names:
reap = reap.where(DockerContainer.name.notin_(seen_names))
await session.execute(reap)
+70
View File
@@ -0,0 +1,70 @@
# plugins/docker/migrations/env.py
"""Alembic env.py for the Docker plugin."""
import asyncio
import os
import sys
from pathlib import Path
from logging.config import fileConfig
from sqlalchemy import pool
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import async_engine_from_config
from alembic import context
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
from steward.models.base import Base
import steward.models # noqa: F401
from plugins.docker.models import DockerContainer, DockerMetric # noqa: F401
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def _get_url() -> str:
import yaml
cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml")
try:
with open(cfg_path) as f:
cfg = yaml.safe_load(f) or {}
url = cfg.get("database", {}).get("url", "")
except FileNotFoundError:
url = ""
return os.environ.get("STEWARD_DATABASE__URL", url)
def run_migrations_offline() -> None:
context.configure(
url=_get_url(), target_metadata=target_metadata,
literal_binds=True, dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def do_run_migrations(connection: Connection) -> None:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
cfg = config.get_section(config.config_ini_section, {})
cfg["sqlalchemy.url"] = _get_url()
connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
asyncio.run(run_async_migrations())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
@@ -0,0 +1,47 @@
"""Docker plugin initial tables
Revision ID: docker_001_initial
Revises: (none — branch off core via depends_on)
Create Date: 2026-03-22
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_001_initial"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = "docker"
depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
def upgrade() -> None:
op.create_table(
"docker_containers",
sa.Column("name", sa.String(255), primary_key=True),
sa.Column("container_id", sa.String(64), nullable=False, server_default=""),
sa.Column("image", sa.String(512), nullable=False, server_default=""),
sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
sa.Column("cpu_pct", sa.Float, nullable=True),
sa.Column("mem_usage_bytes", sa.Integer, nullable=True),
sa.Column("mem_limit_bytes", sa.Integer, nullable=True),
sa.Column("mem_pct", sa.Float, nullable=True),
sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"),
sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_table(
"docker_metrics",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("container_name", sa.String(255), nullable=False, index=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True),
sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"),
sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"),
sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"),
)
def downgrade() -> None:
op.drop_table("docker_metrics")
op.drop_table("docker_containers")
@@ -0,0 +1,94 @@
"""Docker collection goes per-host: add host_id, re-key by (host_id, name)
Collection moved from the central single-socket scrape to the host agent, so
containers are now scoped to the host that reported them. docker_containers is
re-keyed (host_id, name) and docker_metrics gains host_id.
Dev-only posture (rule 122): the old tables only ever held the Steward box's
own containers (a single global namespace), which are disposable — so this
DROP+recreates rather than backfilling a host_id onto orphan rows.
Revision ID: docker_002_host_scoped
Revises: docker_001_initial
Create Date: 2026-06-18
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_002_host_scoped"
down_revision: Union[str, None] = "docker_001_initial"
branch_labels: Union[str, Sequence[str], None] = None
# FK targets hosts.id (created in 0002_core_monitors) — make the edge explicit.
depends_on: Union[str, Sequence[str], None] = ("0002_core_monitors",)
def upgrade() -> None:
op.drop_table("docker_metrics")
op.drop_table("docker_containers")
op.create_table(
"docker_containers",
sa.Column("host_id", sa.String(36),
sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True),
sa.Column("name", sa.String(255), primary_key=True),
sa.Column("container_id", sa.String(64), nullable=False, server_default=""),
sa.Column("image", sa.String(512), nullable=False, server_default=""),
sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
sa.Column("cpu_pct", sa.Float, nullable=True),
sa.Column("mem_usage_bytes", sa.Integer, nullable=True),
sa.Column("mem_limit_bytes", sa.Integer, nullable=True),
sa.Column("mem_pct", sa.Float, nullable=True),
sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"),
sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_table(
"docker_metrics",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("host_id", sa.String(36),
sa.ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False),
sa.Column("container_name", sa.String(255), nullable=False),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"),
sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"),
sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"),
)
op.create_index("ix_docker_metrics_host_id", "docker_metrics", ["host_id"])
op.create_index("ix_docker_metrics_container_name", "docker_metrics",
["container_name"])
op.create_index("ix_docker_metrics_scraped_at", "docker_metrics", ["scraped_at"])
op.create_index("ix_docker_metrics_host_container_time", "docker_metrics",
["host_id", "container_name", "scraped_at"])
def downgrade() -> None:
op.drop_table("docker_metrics")
op.drop_table("docker_containers")
op.create_table(
"docker_containers",
sa.Column("name", sa.String(255), primary_key=True),
sa.Column("container_id", sa.String(64), nullable=False, server_default=""),
sa.Column("image", sa.String(512), nullable=False, server_default=""),
sa.Column("status", sa.String(32), nullable=False, server_default="unknown"),
sa.Column("cpu_pct", sa.Float, nullable=True),
sa.Column("mem_usage_bytes", sa.Integer, nullable=True),
sa.Column("mem_limit_bytes", sa.Integer, nullable=True),
sa.Column("mem_pct", sa.Float, nullable=True),
sa.Column("restart_count", sa.Integer, nullable=False, server_default="0"),
sa.Column("ports_json", sa.Text, nullable=False, server_default="[]"),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_table(
"docker_metrics",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("container_name", sa.String(255), nullable=False, index=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True),
sa.Column("cpu_pct", sa.Float, nullable=False, server_default="0"),
sa.Column("mem_pct", sa.Float, nullable=False, server_default="0"),
sa.Column("mem_usage_bytes", sa.Integer, nullable=False, server_default="0"),
)
@@ -0,0 +1,41 @@
"""Docker container enrichment: health, exit/restart, grouping, I/O counters
Adds the fields the agent (≥1.4.0) now reports per container beyond the basic
state: health status, exit code, OOM flag, compose/swarm grouping labels, and
cumulative network/block I/O counters. Additive columns — no data loss, so no
DROP+recreate needed here.
Revision ID: docker_003_container_enrichment
Revises: docker_002_host_scoped
Create Date: 2026-06-19
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_003_container_enrichment"
down_revision: Union[str, None] = "docker_002_host_scoped"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("docker_containers", sa.Column("health", sa.String(16), nullable=True))
op.add_column("docker_containers", sa.Column("exit_code", sa.Integer, nullable=True))
op.add_column("docker_containers",
sa.Column("oom_killed", sa.Boolean, nullable=False, server_default=sa.false()))
op.add_column("docker_containers", sa.Column("compose_project", sa.String(255), nullable=True))
op.add_column("docker_containers", sa.Column("service_name", sa.String(255), nullable=True))
op.add_column("docker_containers", sa.Column("task_id", sa.String(64), nullable=True))
op.add_column("docker_containers", sa.Column("node_id", sa.String(64), nullable=True))
op.add_column("docker_containers", sa.Column("net_rx_bytes", sa.BigInteger, nullable=True))
op.add_column("docker_containers", sa.Column("net_tx_bytes", sa.BigInteger, nullable=True))
op.add_column("docker_containers", sa.Column("blk_read_bytes", sa.BigInteger, nullable=True))
op.add_column("docker_containers", sa.Column("blk_write_bytes", sa.BigInteger, nullable=True))
def downgrade() -> None:
for col in ("blk_write_bytes", "blk_read_bytes", "net_tx_bytes", "net_rx_bytes",
"node_id", "task_id", "service_name", "compose_project",
"oom_killed", "exit_code", "health"):
op.drop_column("docker_containers", col)
@@ -0,0 +1,77 @@
"""Docker lifecycle events + swarm topology tables
Adds the storage for milestone-77 monitoring depth that doesn't fit on the
per-container row:
* docker_events — lifecycle (start/stop/die/oom/health_change) derived by
diffing consecutive host snapshots; host-scoped, indexed for both timeline
lookups (host_id, container_name, at) and retention pruning (at).
* docker_swarm_services / docker_swarm_nodes — manager-reported Swarm
topology (desired-vs-running replicas, node role/availability/status).
All new tables — purely additive, so no DROP+recreate. `event`/`mode`/`role`
etc. are plain strings (no CHECK whitelist), matching how docker_containers
models `status`; keeps the value sets open without a migration per addition
(so rule 36 doesn't apply here).
Revision ID: docker_004_events_swarm
Revises: docker_003_container_enrichment
Create Date: 2026-06-19
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_004_events_swarm"
down_revision: Union[str, None] = "docker_003_container_enrichment"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"docker_events",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("host_id", sa.String(36),
sa.ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False),
sa.Column("container_name", sa.String(255), nullable=False),
sa.Column("event", sa.String(16), nullable=False),
sa.Column("detail", sa.String(255), nullable=True),
sa.Column("at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_docker_events_host_container_time", "docker_events",
["host_id", "container_name", "at"])
op.create_index("ix_docker_events_at", "docker_events", ["at"])
op.create_table(
"docker_swarm_services",
sa.Column("host_id", sa.String(36),
sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True),
sa.Column("service_name", sa.String(255), primary_key=True),
sa.Column("mode", sa.String(16), nullable=False, server_default="replicated"),
sa.Column("desired", sa.Integer, nullable=False, server_default="0"),
sa.Column("running", sa.Integer, nullable=False, server_default="0"),
sa.Column("image", sa.String(512), nullable=True),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_table(
"docker_swarm_nodes",
sa.Column("host_id", sa.String(36),
sa.ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True),
sa.Column("node_id", sa.String(64), primary_key=True),
sa.Column("hostname", sa.String(255), nullable=False, server_default=""),
sa.Column("role", sa.String(16), nullable=False, server_default="worker"),
sa.Column("availability", sa.String(16), nullable=False, server_default="active"),
sa.Column("status", sa.String(16), nullable=False, server_default="unknown"),
sa.Column("leader", sa.Boolean, nullable=False, server_default=sa.false()),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
op.drop_table("docker_swarm_nodes")
op.drop_table("docker_swarm_services")
op.drop_index("ix_docker_events_at", table_name="docker_events")
op.drop_index("ix_docker_events_host_container_time", table_name="docker_events")
op.drop_table("docker_events")
@@ -0,0 +1,29 @@
"""Docker swarm service placement column
Adds docker_swarm_services.placement_json — the task→node placement of a
service's running replicas, captured from the agent's swarm payload (a manager
sees every task, so this records cross-node placement the local container rows
can't). Additive column; no DROP+recreate.
Revision ID: docker_005_swarm_placement
Revises: docker_004_events_swarm
Create Date: 2026-06-19
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_005_swarm_placement"
down_revision: Union[str, None] = "docker_004_events_swarm"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("docker_swarm_services",
sa.Column("placement_json", sa.Text, nullable=False,
server_default="[]"))
def downgrade() -> None:
op.drop_column("docker_swarm_services", "placement_json")
@@ -0,0 +1,45 @@
"""Docker hourly metric rollup table
Adds docker_metrics_hourly — the coarse series that retention rolls raw
docker_metrics into before pruning them, so multi-day history stays cheap.
One row per (host, container, hour bucket); the unique constraint is the
conflict target for the idempotent rollup upsert. Additive create_table.
Revision ID: docker_006_metric_rollup
Revises: docker_005_swarm_placement
Create Date: 2026-06-19
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_006_metric_rollup"
down_revision: Union[str, None] = "docker_005_swarm_placement"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"docker_metrics_hourly",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("host_id", sa.String(length=36), nullable=False),
sa.Column("container_name", sa.String(length=255), nullable=False),
sa.Column("bucket", sa.DateTime(timezone=True), nullable=False),
sa.Column("cpu_pct", sa.Float(), nullable=False, server_default="0"),
sa.Column("mem_pct", sa.Float(), nullable=False, server_default="0"),
sa.Column("mem_usage_bytes", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("sample_count", sa.Integer(), nullable=False, server_default="0"),
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("host_id", "container_name", "bucket",
name="uq_docker_metrics_hourly_bucket"),
)
op.create_index("ix_docker_metrics_hourly_bucket",
"docker_metrics_hourly", ["bucket"])
def downgrade() -> None:
op.drop_index("ix_docker_metrics_hourly_bucket",
table_name="docker_metrics_hourly")
op.drop_table("docker_metrics_hourly")
@@ -0,0 +1,55 @@
"""Docker disk usage + image storage tables
Adds docker_disk_usage (one per-host /system/df summary row) and docker_images
(the heavy-hitter image storage records). Both current-state, host-scoped,
re-reported each interval. Additive create_table.
Revision ID: docker_007_disk_usage
Revises: docker_006_metric_rollup
Create Date: 2026-06-19
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_007_disk_usage"
down_revision: Union[str, None] = "docker_006_metric_rollup"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"docker_disk_usage",
sa.Column("host_id", sa.String(length=36), nullable=False),
sa.Column("layers_size", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("images_size", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("images_reclaimable", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("containers_size", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("volumes_size", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("build_cache_size", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("images_total", sa.Integer(), nullable=False, server_default="0"),
sa.Column("images_active", sa.Integer(), nullable=False, server_default="0"),
sa.Column("containers_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("volumes_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("host_id"),
)
op.create_table(
"docker_images",
sa.Column("host_id", sa.String(length=36), nullable=False),
sa.Column("image_id", sa.String(length=64), nullable=False),
sa.Column("repo_tag", sa.String(length=512), nullable=False, server_default="<none>"),
sa.Column("size", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("shared_size", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("containers", sa.Integer(), nullable=False, server_default="0"),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("host_id", "image_id"),
)
def downgrade() -> None:
op.drop_table("docker_images")
op.drop_table("docker_disk_usage")
@@ -0,0 +1,33 @@
"""Widen memory byte columns to BIGINT
docker_metrics.mem_usage_bytes and docker_containers.mem_usage_bytes /
mem_limit_bytes were int4, which overflows for any container using >2.1 GB
(asyncpg: "value out of int32 range"), failing the whole ingest batch. Widen to
BIGINT — a safe in-place int4→int8 promotion, no data rewrite.
Revision ID: docker_008_bigint_mem
Revises: docker_007_disk_usage
Create Date: 2026-06-20
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_008_bigint_mem"
down_revision: Union[str, None] = "docker_007_disk_usage"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.alter_column("docker_metrics", "mem_usage_bytes", type_=sa.BigInteger())
op.alter_column("docker_containers", "mem_usage_bytes", type_=sa.BigInteger())
op.alter_column("docker_containers", "mem_limit_bytes", type_=sa.BigInteger())
def downgrade() -> None:
# Narrowing back to int4 will error if any stored value exceeds 2^31 — that's
# the very condition this migration fixed, so downgrade is best-effort.
op.alter_column("docker_containers", "mem_limit_bytes", type_=sa.Integer())
op.alter_column("docker_containers", "mem_usage_bytes", type_=sa.Integer())
op.alter_column("docker_metrics", "mem_usage_bytes", type_=sa.Integer())
+264
View File
@@ -0,0 +1,264 @@
# plugins/docker/models.py
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from sqlalchemy import (
BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text,
UniqueConstraint,
)
from sqlalchemy.orm import Mapped, mapped_column
from steward.models.base import Base
class DockerContainer(Base):
"""Latest known state per container, scoped to the host that reported it.
Collection is per-host via the host agent, so container names are only
unique within a host — the natural key is (host_id, name). host_id is NOT
NULL: every container arrives through a host_agent ingest that resolves a
Host first. Deleting a host cascades its containers away.
"""
__tablename__ = "docker_containers"
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
)
name: Mapped[str] = mapped_column(String(255), primary_key=True)
container_id: Mapped[str] = mapped_column(String(64), nullable=False, default="")
image: Mapped[str] = mapped_column(String(512), nullable=False, default="")
status: Mapped[str] = mapped_column(String(32), nullable=False, default="unknown")
# running | stopped | paused | exited | dead
cpu_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
# BigInteger: a container's memory usage/limit routinely exceeds 2^31 bytes
# (~2.1 GB), which overflows int4 and fails the insert.
mem_usage_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
mem_limit_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
mem_pct: Mapped[float | None] = mapped_column(Float, nullable=True)
restart_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
ports_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
# JSON: [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}]
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
scraped_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
)
# ── Enrichment (agent ≥ 1.4.0) ────────────────────────────────────────────
# Health/exit/restart come from `docker inspect` (not the list endpoint);
# exit_code is only meaningful for stopped containers.
health: Mapped[str | None] = mapped_column(String(16), nullable=True) # healthy|unhealthy|starting
exit_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
oom_killed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
# Grouping: compose project + swarm placement, read off container labels.
compose_project: Mapped[str | None] = mapped_column(String(255), nullable=True)
service_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
task_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
node_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
# Cumulative-since-start I/O counters (BigInteger — they exceed 2^31 quickly).
net_rx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
net_tx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
blk_read_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
blk_write_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
class DockerMetric(Base):
"""Time-series CPU/memory per container — one row per sample per running
container, scoped to the reporting host."""
__tablename__ = "docker_metrics"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
)
container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
scraped_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
index=True,
)
cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
# BigInteger: per-container memory usage exceeds 2^31 bytes (~2.1 GB) routinely.
mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
# Per-container history lookups filter on (host_id, container_name) then sort
# by time — a composite index keeps the rows() sparkline queries cheap.
__table_args__ = (
Index("ix_docker_metrics_host_container_time",
"host_id", "container_name", "scraped_at"),
)
class DockerMetricHourly(Base):
"""Hourly rollup of docker_metrics — avg cpu/mem per container per hour.
Raw per-sample rows (~2880/container/day at 30s) are pruned beyond a short
window; before deletion they're aggregated here so multi-day history stays
cheap to store and query. One row per (host, container, hour bucket); the
unique constraint lets retention upsert idempotently if it re-runs before the
raw rows are deleted. `bucket` is the hour-truncated sample time.
"""
__tablename__ = "docker_metrics_hourly"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
)
container_name: Mapped[str] = mapped_column(String(255), nullable=False)
bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
__table_args__ = (
# One bucket per container per host — the conflict target for the
# idempotent rollup upsert; doubles as the history-query index.
UniqueConstraint("host_id", "container_name", "bucket",
name="uq_docker_metrics_hourly_bucket"),
Index("ix_docker_metrics_hourly_bucket", "bucket"),
)
class DockerEvent(Base):
"""Lifecycle events derived by diffing consecutive host snapshots.
The agent only reports current state, so Steward synthesises lifecycle by
comparing each new snapshot against the stored DockerContainer state for the
host: a container that appears → `start`; one that vanishes → `stop`; a
transition to a stopped status with a non-zero exit → `die`; an OOM-kill
flip → `oom`; a health status change → `health_change`. Scoped to the
reporting host (names are only unique within a host). `event` is a plain
string (no CHECK whitelist), matching how `status` is modelled on
DockerContainer — keeps the value set open without a migration per addition.
"""
__tablename__ = "docker_events"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
)
container_name: Mapped[str] = mapped_column(String(255), nullable=False)
event: Mapped[str] = mapped_column(String(16), nullable=False)
# start | stop | die | oom | health_change
detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
# e.g. "exit 137", "healthy→unhealthy", image on start
at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
)
# Timeline lookups filter on (host_id, container_name) and sort by time;
# retention prunes by `at` alone — index both access paths.
__table_args__ = (
Index("ix_docker_events_host_container_time",
"host_id", "container_name", "at"),
Index("ix_docker_events_at", "at"),
)
class DockerSwarmService(Base):
"""A Swarm service as seen by a manager host: desired vs running replicas.
Manager-scoped — only a host that is a Swarm manager reports these, so a
single-manager cluster yields one row set keyed by that host. Service names
are unique within a swarm; we still key by (host_id, service_name) so two
managers reporting independently don't collide.
"""
__tablename__ = "docker_swarm_services"
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
)
service_name: Mapped[str] = mapped_column(String(255), primary_key=True)
mode: Mapped[str] = mapped_column(String(16), nullable=False, default="replicated")
# replicated | global
desired: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
running: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
image: Mapped[str | None] = mapped_column(String(512), nullable=True)
# task→node placement of running replicas: [{"node_id", "running"}], JSON.
# A manager sees every task, so this captures cross-node placement that the
# local docker_containers rows (this host only) can't.
placement_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
)
class DockerDiskUsage(Base):
"""Per-host Docker disk usage summary from `/system/df` (current state).
One row per host (the agent re-reports the full summary each interval).
Reclaimable = bytes held by images no container references. Sizes are bytes
(BigInteger — image caches exceed 2^31 easily).
"""
__tablename__ = "docker_disk_usage"
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
)
layers_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
images_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
images_reclaimable: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
containers_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
volumes_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
build_cache_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
images_total: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
images_active: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
containers_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
volumes_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
)
class DockerImage(Base):
"""Per-host image storage record (the heavy hitters from `/system/df`).
Keyed (host_id, image_id); `containers` is the reference count (0 ⇒ the
image's size is reclaimable). Replaced wholesale per host on each report.
"""
__tablename__ = "docker_images"
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
)
image_id: Mapped[str] = mapped_column(String(64), primary_key=True)
repo_tag: Mapped[str] = mapped_column(String(512), nullable=False, default="<none>")
size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
shared_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
containers: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
)
class DockerSwarmNode(Base):
"""A Swarm node as seen by a manager host: role / availability / status."""
__tablename__ = "docker_swarm_nodes"
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
)
node_id: Mapped[str] = mapped_column(String(64), primary_key=True)
hostname: Mapped[str] = mapped_column(String(255), nullable=False, default="")
role: Mapped[str] = mapped_column(String(16), nullable=False, default="worker")
# manager | worker
availability: Mapped[str] = mapped_column(String(16), nullable=False, default="active")
# active | pause | drain
status: Mapped[str] = mapped_column(String(16), nullable=False, default="unknown")
# ready | down | unknown
leader: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
)
+16
View File
@@ -0,0 +1,16 @@
name: docker
version: "2.0.0"
description: "Per-host Docker container status + resource usage, collected by the host agent"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker"
tags:
- containers
- docker
- infrastructure
# No config: collection is agent-driven (the host agent reads each host's local
# socket and pushes containers to the host_agent ingest). This plugin is pure
# presentation + storage, so there's no socket path or scrape interval to tune.
+128
View File
@@ -0,0 +1,128 @@
# plugins/docker/retention.py
"""Bound Docker time-series growth: roll up old metrics, prune old rows.
Published as the "docker.run_retention" capability (see __init__.setup) so the
core cleanup task can drive it WITHOUT importing the docker models (same
opportunistic-coupling pattern as docker.persist_host_samples). Runs inside the
caller's open transaction; never opens or commits its own.
The scaling concern is docker_metrics: ~2880 rows/container/day at a 30s sample.
We keep raw samples for a short window, then aggregate everything older into
hourly averages (docker_metrics_hourly) and delete the raw rows — so multi-day
history stays cheap to store and query. docker_events is light but unbounded
without a cutoff, so it gets a (longer) window too.
"""
from __future__ import annotations
from datetime import datetime, timedelta
def _hour_floor(dt: datetime) -> datetime:
"""Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
return dt.replace(minute=0, second=0, microsecond=0)
def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
"""Hour-aligned boundary below which raw metrics get rolled up + deleted.
Aligning to the hour means we only ever roll up *whole* elapsed hours — a
bucket is never split across the keep/roll boundary, so re-running can't
produce a partial-then-complete duplicate for the same hour.
"""
return _hour_floor(now - timedelta(days=raw_days))
async def run_docker_retention(
session,
*,
events_days: int,
metrics_raw_days: int,
metrics_rollup_days: int,
now: datetime | None = None,
) -> dict:
"""Roll up + prune Docker time-series. Returns a counts dict for logging.
1. Aggregate docker_metrics older than the (hour-aligned) raw window into
docker_metrics_hourly (avg cpu/mem per container per hour), upserting so a
re-run is idempotent, then delete those raw rows.
2. Prune rolled-up rows older than the rollup window.
3. Prune docker_events older than the events window.
"""
from datetime import timezone
from sqlalchemy import delete, func, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from .models import DockerEvent, DockerMetric, DockerMetricHourly
if now is None:
now = datetime.now(timezone.utc)
rolled = rolled_rows = events_pruned = rollup_pruned = 0
# ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
raw_cutoff = _rollup_cutoff(now, metrics_raw_days)
hour = func.date_trunc("hour", DockerMetric.scraped_at)
agg = (
select(
DockerMetric.host_id,
DockerMetric.container_name,
hour.label("bucket"),
func.avg(DockerMetric.cpu_pct).label("cpu_pct"),
func.avg(DockerMetric.mem_pct).label("mem_pct"),
func.avg(DockerMetric.mem_usage_bytes).label("mem_usage_bytes"),
func.count().label("sample_count"),
)
.where(DockerMetric.scraped_at < raw_cutoff)
.group_by(DockerMetric.host_id, DockerMetric.container_name, hour)
)
for r in (await session.execute(agg)).all():
stmt = (
pg_insert(DockerMetricHourly)
.values(
host_id=r.host_id,
container_name=r.container_name,
bucket=r.bucket,
cpu_pct=float(r.cpu_pct or 0.0),
mem_pct=float(r.mem_pct or 0.0),
mem_usage_bytes=int(r.mem_usage_bytes or 0),
sample_count=int(r.sample_count or 0),
)
.on_conflict_do_update(
constraint="uq_docker_metrics_hourly_bucket",
set_={
"cpu_pct": float(r.cpu_pct or 0.0),
"mem_pct": float(r.mem_pct or 0.0),
"mem_usage_bytes": int(r.mem_usage_bytes or 0),
"sample_count": int(r.sample_count or 0),
},
)
)
await session.execute(stmt)
rolled += 1
rolled_rows += int(r.sample_count or 0)
if rolled:
await session.execute(
delete(DockerMetric).where(DockerMetric.scraped_at < raw_cutoff)
)
# ── 2. Prune rolled-up rows beyond the rollup window ──
rollup_cutoff = now - timedelta(days=metrics_rollup_days)
res = await session.execute(
delete(DockerMetricHourly).where(DockerMetricHourly.bucket < rollup_cutoff)
)
rollup_pruned = res.rowcount or 0
# ── 3. Prune lifecycle events beyond the events window ──
events_cutoff = now - timedelta(days=events_days)
res = await session.execute(
delete(DockerEvent).where(DockerEvent.at < events_cutoff)
)
events_pruned = res.rowcount or 0
return {
"buckets_rolled": rolled,
"raw_rows_rolled": rolled_rows,
"rollup_pruned": rollup_pruned,
"events_pruned": events_pruned,
}
+565
View File
@@ -0,0 +1,565 @@
# plugins/docker/routes.py
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import func, select
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.models.hosts import Host
from steward.core.time_range import parse_range, DEFAULT_RANGE
from .dedup import dedup_by_container_id
from .swarm_view import build_swarm_services
from .models import (
DockerContainer, DockerDiskUsage, DockerEvent, DockerImage, DockerMetric,
DockerSwarmNode, DockerSwarmService,
)
docker_bp = Blueprint("docker", __name__, template_folder="templates")
def _human_bytes(n: int | None) -> str:
"""Compact binary size string (e.g. '1.4 GiB', '512 MiB', '0 B')."""
if n is None:
return ""
size = float(n)
for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
if abs(size) < 1024.0 or unit == "TiB":
if unit == "B":
return f"{int(size)} {unit}"
return f"{size:.1f} {unit}"
size /= 1024.0
return f"{size:.1f} TiB"
def _human_uptime(started_at: datetime | None) -> str | None:
"""Compact 'how long running' string (e.g. '3d 4h', '5h 12m', '8m')."""
if started_at is None:
return None
if started_at.tzinfo is None:
started_at = started_at.replace(tzinfo=timezone.utc)
secs = int((datetime.now(timezone.utc) - started_at).total_seconds())
if secs < 0:
return None
d, rem = divmod(secs, 86400)
h, rem = divmod(rem, 3600)
m, _ = divmod(rem, 60)
if d:
return f"{d}d {h}h"
if h:
return f"{h}h {m}m"
return f"{m}m"
def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
if len(values) < 2:
return f'<svg width="{width}" height="{height}"></svg>'
mn, mx = min(values), max(values)
if mx == mn:
mx = mn + 1.0
step = width / (len(values) - 1)
pts = []
for i, v in enumerate(values):
x = i * step
y = height - (v - mn) / (mx - mn) * (height - 2) - 1
pts.append(f"{x:.1f},{y:.1f}")
poly = " ".join(pts)
return (
f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
f'style="vertical-align:middle;">'
f'<polyline points="{poly}" fill="none" stroke="#6060c0" stroke-width="1.5"/>'
f'</svg>'
)
def _bucket(values: list[float], target: int = 40) -> list[float]:
"""Bucket-average a series down to ~target points (DB-agnostic, in Python).
Replaces the old SQL strftime() bucketing, which was SQLite-only and broke
on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of
rows — averaging into a readable point count keeps the sparkline's shape.
"""
n = len(values)
if n <= target:
return values
size = (n + target - 1) // target
out: list[float] = []
for i in range(0, n, size):
chunk = values[i:i + size]
out.append(sum(chunk) / len(chunk))
return out
async def _container_history(db, host_id: str, name: str, since) -> tuple[list, list]:
"""Return (cpu_series, mem_series) sparkline-ready for one host's container."""
rows = (await db.execute(
select(DockerMetric.cpu_pct, DockerMetric.mem_pct)
.where(DockerMetric.host_id == host_id)
.where(DockerMetric.container_name == name)
.where(DockerMetric.scraped_at >= since)
.order_by(DockerMetric.scraped_at)
)).all()
cpu = _bucket([r.cpu_pct or 0.0 for r in rows])
mem = _bucket([r.mem_pct or 0.0 for r in rows])
return cpu, mem
@docker_bp.get("/")
@require_role(UserRole.viewer)
async def index():
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
current_range = request.args.get("range", DEFAULT_RANGE)
# Show the Swarm link only when a manager has actually reported topology, so
# non-swarm installs aren't offered an always-empty page.
async with current_app.db_sessionmaker() as db:
has_swarm = (await db.execute(
select(DockerSwarmService.host_id).limit(1))).first() is not None
has_disk = (await db.execute(
select(DockerDiskUsage.host_id).limit(1))).first() is not None
return await render_template(
"docker/index.html",
poll_interval=poll_interval,
current_range=current_range,
has_swarm=has_swarm,
has_disk=has_disk,
)
async def _host_map(db, host_ids: set[str]) -> dict[str, Host]:
if not host_ids:
return {}
return {
h.id: h for h in (await db.execute(
select(Host).where(Host.id.in_(host_ids)))).scalars().all()
}
@docker_bp.get("/rows")
@require_role(UserRole.viewer)
async def rows():
"""HTMX fragment: swarm services (collapsed, cluster-complete) on top, then
non-swarm containers grouped by host with resource sparklines."""
since, range_key = parse_range(request.args.get("range"))
async with current_app.db_sessionmaker() as db:
containers = list((await db.execute(
select(DockerContainer).order_by(
(DockerContainer.status == "running").desc(),
DockerContainer.name,
)
)).scalars())
services = list((await db.execute(select(DockerSwarmService))).scalars())
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
hosts = await _host_map(
db,
{c.host_id for c in containers}
| {n.host_id for n in nodes} | {s.host_id for s in services},
)
# Swarm tasks collapse into per-service rows (each replica labelled with
# its host, plus "ghost" replicas for tasks on agent-less nodes, drawn
# from the managers' placement). Non-swarm containers keep the host view.
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
host_name = {h.id: h.name for h in hosts.values()}
swarm_view = build_swarm_services(containers, services, node_hostname, host_name)
standalone = [c for c in containers if not c.service_name]
groups: dict[str, dict] = {}
for c in standalone:
g = groups.get(c.host_id)
if g is None:
host = hosts.get(c.host_id)
g = groups[c.host_id] = {
"host": host,
"host_id": c.host_id,
"host_name": host.name if host else c.host_id,
"containers": [], "running": 0, "stopped": 0,
}
cpu_hist, mem_hist = await _container_history(db, c.host_id, c.name, since)
g["containers"].append({
"container": c,
"ports": json.loads(c.ports_json) if c.ports_json else [],
"uptime": _human_uptime(c.started_at) if c.status == "running" else None,
"sparkline_cpu": _sparkline(cpu_hist),
"sparkline_mem": _sparkline(mem_hist),
})
if c.status == "running":
g["running"] += 1
else:
g["stopped"] += 1
# Sub-group each host's containers by compose project, preserving the
# running-first ordering. Containers with no project fall into an unlabelled
# bucket rendered flat, so plain hosts look unchanged.
for g in groups.values():
subs: dict[str, dict] = {}
order: list[str] = []
for item in g["containers"]:
c = item["container"]
label = c.compose_project or None
key = label or ""
if key not in subs:
subs[key] = {"label": label, "containers": []}
order.append(key)
subs[key]["containers"].append(item)
g["subgroups"] = [subs[k] for k in order]
g["grouped"] = any(s["label"] for s in g["subgroups"])
host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
swarm_services = swarm_view["services"]
standalone_running = sum(g["running"] for g in host_groups)
ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"])
return await render_template(
"docker/rows.html",
host_groups=host_groups,
swarm_services=swarm_services,
running=standalone_running + sum(s["running"] for s in swarm_services),
stopped=len(standalone) - standalone_running,
total=len(containers) + ghost_total,
range_key=range_key,
)
@docker_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""HTMX dashboard widget: container status overview, grouped by host."""
show_stopped = request.args.get("show_stopped", "no") == "yes"
widget_id = request.args.get("wid", "0")
since_24h = datetime.now(timezone.utc) - timedelta(hours=24)
async with current_app.db_sessionmaker() as db:
all_containers = dedup_by_container_id(list((await db.execute(
select(DockerContainer).order_by(
(DockerContainer.status == "running").desc(),
DockerContainer.name,
)
)).scalars()))
services = list((await db.execute(select(DockerSwarmService))).scalars())
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
# Recent failures are more actionable than a static "stopped" tally. Count
# DISTINCT container names so the two managers' duplicate die/oom events
# don't double it.
failed_24h = (await db.execute(
select(func.count(func.distinct(DockerEvent.container_name)))
.where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since_24h)
)).scalar() or 0
hosts = await _host_map(
db,
{c.host_id for c in all_containers}
| {n.host_id for n in nodes} | {s.host_id for s in services},
)
# Swarm tasks collapse into per-service rows (each replica tagged with its
# host); non-swarm containers stay grouped by host.
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
host_name = {h.id: h.name for h in hosts.values()}
swarm_services = build_swarm_services(all_containers, services, node_hostname, host_name)["services"]
standalone = [c for c in all_containers if not c.service_name]
running = [c for c in standalone if c.status == "running"]
display = standalone if show_stopped else running
# Group for display so multi-host fleets read clearly; single-host stays flat.
groups: dict[str, dict] = {}
for c in display:
g = groups.setdefault(c.host_id, {
"host_id": c.host_id,
"host_name": host_name.get(c.host_id, c.host_id),
"containers": [],
})
g["containers"].append(c)
host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
return await render_template(
"docker/widget.html",
host_groups=host_groups,
multi_host=len(host_groups) > 1,
swarm_services=swarm_services,
running_count=len(running) + sum(s["running"] for s in swarm_services),
failed_24h=failed_24h,
total_count=len(all_containers) + len(swarm_services),
show_stopped=show_stopped,
widget_id=widget_id,
)
@docker_bp.get("/widget/resources")
@require_role(UserRole.viewer)
async def widget_resources():
"""HTMX dashboard widget: CPU + memory usage for the busiest containers."""
limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
widget_id = request.args.get("wid", "0")
async with current_app.db_sessionmaker() as db:
# Dedup before the limit, else swarm tasks reported by both managers can
# fill the list with duplicates of the same container.
containers = dedup_by_container_id(list((await db.execute(
select(DockerContainer)
.where(DockerContainer.status == "running")
.order_by(DockerContainer.cpu_pct.desc().nullslast())
)).scalars()))[:limit]
hosts = await _host_map(db, {c.host_id for c in containers})
rows_data = [
{"c": c, "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id}
for c in containers
]
return await render_template(
"docker/widget_resources.html",
rows=rows_data,
multi_host=len({c.host_id for c in containers}) > 1,
widget_id=widget_id,
)
@docker_bp.get("/host/<host_id>")
@require_role(UserRole.viewer)
async def host_panel(host_id: str):
"""Per-host Docker fragment embedded on the core Hosts hub via HTMX.
Renders nothing when the host reports no containers, so hosts without Docker
don't carry an empty card.
"""
async with current_app.db_sessionmaker() as db:
containers = list((await db.execute(
select(DockerContainer)
.where(DockerContainer.host_id == host_id)
.order_by(
(DockerContainer.status == "running").desc(),
DockerContainer.name,
)
)).scalars())
if not containers:
return ""
running = sum(1 for c in containers if c.status == "running")
return await render_template(
"docker/host_panel.html",
containers=containers,
running=running,
stopped=len(containers) - running,
)
# Glyph + colour per lifecycle event, so the timeline reads at a glance.
_EVENT_STYLE = {
"start": ("", "var(--green)"),
"stop": ("", "var(--text-muted)"),
"die": ("", "var(--red)"),
"oom": ("", "var(--red)"),
"health_change": ("", "var(--orange)"),
}
@docker_bp.get("/container/<host_id>/<name>")
@require_role(UserRole.viewer)
async def container_detail(host_id: str, name: str):
"""Full detail page for one container: facts, resource history, lifecycle."""
async with current_app.db_sessionmaker() as db:
c = await db.get(DockerContainer, (host_id, name))
host = await db.get(Host, host_id)
events = []
if c is not None:
events = list((await db.execute(
select(DockerEvent)
.where(DockerEvent.host_id == host_id)
.where(DockerEvent.container_name == name)
.order_by(DockerEvent.at.desc())
.limit(50)
)).scalars())
if c is None:
return await render_template(
"docker/container_detail.html",
container=None, host_id=host_id, name=name,
), 404
return await render_template(
"docker/container_detail.html",
container=c,
host=host,
host_id=host_id,
name=name,
ports=json.loads(c.ports_json) if c.ports_json else [],
uptime=_human_uptime(c.started_at) if c.status == "running" else None,
net_rx=_human_bytes(c.net_rx_bytes), net_tx=_human_bytes(c.net_tx_bytes),
blk_read=_human_bytes(c.blk_read_bytes), blk_write=_human_bytes(c.blk_write_bytes),
events=[
{"event": e.event, "detail": e.detail, "at": e.at,
"glyph": _EVENT_STYLE.get(e.event, ("", "var(--text-muted)"))[0],
"colour": _EVENT_STYLE.get(e.event, ("", "var(--text-muted)"))[1]}
for e in events
],
current_range=request.args.get("range", DEFAULT_RANGE),
)
@docker_bp.get("/swarm")
@require_role(UserRole.viewer)
async def swarm():
"""Swarm topology page: services with replica health, nodes, placement.
Swarm services/nodes are cluster-global — every manager's API returns the
same list — but each manager reports them independently (rows keyed by
host_id). So we group the reporting managers into swarms (managers that share
any node_id are the same cluster) and dedup within each: one row per service
(by name) and per node (by node_id), keeping the freshest. Without this, two
managers in one cluster list every service/node twice.
"""
async with current_app.db_sessionmaker() as db:
services = list((await db.execute(
select(DockerSwarmService).order_by(DockerSwarmService.service_name)
)).scalars())
nodes = list((await db.execute(
select(DockerSwarmNode).order_by(
DockerSwarmNode.role, DockerSwarmNode.hostname)
)).scalars())
hosts = await _host_map(db, {s.host_id for s in services} | {n.host_id for n in nodes})
# ── Group reporting managers into swarms by shared node membership ──────────
manager_ids = {s.host_id for s in services} | {n.host_id for n in nodes}
nodes_by_mgr: dict[str, set[str]] = {}
for n in nodes:
nodes_by_mgr.setdefault(n.host_id, set()).add(n.node_id)
parent = {m: m for m in manager_ids}
def _find(x: str) -> str:
while parent[x] != x:
parent[x] = parent[parent[x]]
x = parent[x]
return x
mgrs = list(manager_ids)
for i in range(len(mgrs)):
for j in range(i + 1, len(mgrs)):
a, b = mgrs[i], mgrs[j]
if nodes_by_mgr.get(a) and nodes_by_mgr.get(b) and (nodes_by_mgr[a] & nodes_by_mgr[b]):
parent[_find(a)] = _find(b)
swarm_members: dict[str, set[str]] = {}
for m in manager_ids:
swarm_members.setdefault(_find(m), set()).add(m)
# ── Dedup services (by name) + nodes (by node_id) within each swarm ─────────
swarms = []
for members in swarm_members.values():
svc_by_name: dict[str, DockerSwarmService] = {}
for s in services:
if s.host_id in members and (
s.service_name not in svc_by_name
or s.updated_at > svc_by_name[s.service_name].updated_at
):
svc_by_name[s.service_name] = s
node_by_id: dict[str, DockerSwarmNode] = {}
for n in nodes:
if n.host_id in members and (
n.node_id not in node_by_id
or n.updated_at > node_by_id[n.node_id].updated_at
):
node_by_id[n.node_id] = n
node_names = {nid: (nrow.hostname or nid) for nid, nrow in node_by_id.items()}
svc_dicts = []
for s in sorted(svc_by_name.values(), key=lambda s: s.service_name):
placement = []
for p in (json.loads(s.placement_json) if s.placement_json else []):
nid = p.get("node_id", "")
placement.append({
"node": node_names.get(nid, nid[:12] or "?"),
"running": p.get("running", 0),
})
svc_dicts.append({
"name": s.service_name, "mode": s.mode,
"running": s.running, "desired": s.desired, "image": s.image,
"healthy": s.running >= s.desired and s.desired > 0,
"degraded": 0 < s.running < s.desired,
"down": s.running == 0 and s.desired > 0,
"placement": placement,
})
node_rows = sorted(
node_by_id.values(), key=lambda n: (n.role, (n.hostname or n.node_id).lower())
)
manager_names = sorted(hosts[m].name if m in hosts else m for m in members)
swarms.append({
"managers": manager_names,
"services": svc_dicts,
"nodes": node_rows,
})
swarms.sort(key=lambda g: g["managers"][0].lower() if g["managers"] else "")
return await render_template("docker/swarm.html", swarms=swarms)
@docker_bp.get("/disk")
@require_role(UserRole.viewer)
async def disk():
"""Image/disk usage page: reclaimable space, per-image sizes, stopped count.
Read-only — prune actions are deferred to the cleanup-actions milestone, so
this surfaces the numbers and notes where reclaim lives.
"""
async with current_app.db_sessionmaker() as db:
summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
images = list((await db.execute(
select(DockerImage).order_by(DockerImage.size.desc())
)).scalars())
# Stopped-container count per host, surfaced alongside reclaimable space.
stopped_rows = (await db.execute(
select(DockerContainer.host_id)
.where(DockerContainer.status != "running")
)).scalars()
stopped_by_host: dict[str, int] = {}
for hid in stopped_rows:
stopped_by_host[hid] = stopped_by_host.get(hid, 0) + 1
hosts = await _host_map(db, {s.host_id for s in summaries})
images_by_host: dict[str, list] = {}
for im in images:
images_by_host.setdefault(im.host_id, []).append({
"repo_tag": im.repo_tag, "size": _human_bytes(im.size),
"shared": _human_bytes(im.shared_size), "containers": im.containers,
"reclaimable": im.containers == 0,
})
host_groups = [
{
"host_id": s.host_id,
"host_name": hosts[s.host_id].name if s.host_id in hosts else s.host_id,
"summary": {
"images_total": s.images_total, "images_active": s.images_active,
"images_size": _human_bytes(s.images_size),
"images_reclaimable": _human_bytes(s.images_reclaimable),
"layers_size": _human_bytes(s.layers_size),
"containers_count": s.containers_count,
"containers_size": _human_bytes(s.containers_size),
"volumes_count": s.volumes_count,
"volumes_size": _human_bytes(s.volumes_size),
"build_cache_size": _human_bytes(s.build_cache_size),
},
"stopped": stopped_by_host.get(s.host_id, 0),
"images": images_by_host.get(s.host_id, []),
}
for s in summaries
]
host_groups.sort(key=lambda g: g["host_name"].lower())
return await render_template("docker/disk.html", host_groups=host_groups)
@docker_bp.get("/container/<host_id>/<name>/history")
@require_role(UserRole.viewer)
async def container_history(host_id: str, name: str):
"""HTMX fragment: CPU + memory sparklines for the selected time range."""
since, range_key = parse_range(request.args.get("range"))
async with current_app.db_sessionmaker() as db:
cpu_hist, mem_hist = await _container_history(db, host_id, name, since)
return await render_template(
"docker/_container_history.html",
sparkline_cpu=_sparkline(cpu_hist, width=320, height=48),
sparkline_mem=_sparkline(mem_hist, width=320, height=48),
have_data=len(cpu_hist) >= 2,
range_key=range_key,
)
+140
View File
@@ -0,0 +1,140 @@
"""Model-free builder for the swarm-aware container view.
Merges two sources into one service-grouped, cluster-complete picture:
• docker_containers — real container rows, collected LOCAL-per-node, so each
carries the host (= node) it runs on plus its swarm service/node labels.
• DockerSwarmService.placement_json — the managers' cluster-wide task→node
counts, which include tasks on nodes that run no Steward agent (and so have
no container row of their own).
For every swarm service we list the real replicas we have detail for, then add
"ghost" replicas for the remaining placement count on each node (agent-less
nodes). Non-swarm containers pass through grouped by host, unchanged.
Kept model-free (no ORM imports) so it's unit-testable and safe to import in
tests without the plugin-loader "table already defined" gotcha.
"""
from __future__ import annotations
import json
def _service_state(running: int, desired: int) -> str:
if desired > 0 and running >= desired:
return "healthy"
if running > 0:
return "degraded"
return "down"
def _placement(service) -> list[dict]:
raw = getattr(service, "placement_json", None) or "[]"
try:
data = json.loads(raw)
except (ValueError, TypeError):
return []
return [p for p in data if isinstance(p, dict)]
def build_swarm_services(containers, services, node_hostname: dict, host_name: dict) -> dict:
"""Return {"services": [...], "standalone": [...]}.
services[i] = {name, mode, desired, running, state, replicas:[...]}
real replica = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
health, restart_count}
ghost replica = {ghost:True, host, count} # placement with no local row
standalone[i] = {host_id, host, containers:[...]} # non-swarm, by host
"""
containers = list(containers)
swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
standalone_cs = [c for c in containers if not getattr(c, "service_name", None)]
# Dedup service rows by name (every manager reports the same cluster-global
# set); keep the first — placement is identical across managers.
svc_by_name: dict[str, object] = {}
for s in services:
name = getattr(s, "service_name", None)
if name and name not in svc_by_name:
svc_by_name[name] = s
# Real containers grouped by service name.
cs_by_service: dict[str, list] = {}
for c in swarm_cs:
cs_by_service.setdefault(c.service_name, []).append(c)
def _host_of(c):
return (host_name.get(getattr(c, "host_id", None))
or node_hostname.get(getattr(c, "node_id", None))
or getattr(c, "host_id", None) or "?")
out_services = []
for name in sorted(set(svc_by_name) | set(cs_by_service)):
svc = svc_by_name.get(name)
reals = cs_by_service.get(name, [])
replicas = []
for c in sorted(reals, key=lambda c: (_host_of(c).lower(), c.name)):
replicas.append({
"ghost": False,
"host": _host_of(c),
"host_id": getattr(c, "host_id", None),
"name": c.name,
"status": getattr(c, "status", "unknown"),
"cpu_pct": getattr(c, "cpu_pct", None),
"mem_pct": getattr(c, "mem_pct", None),
"health": getattr(c, "health", None),
"restart_count": getattr(c, "restart_count", 0) or 0,
})
# Real running replicas per node, to subtract from placement.
real_running_by_node: dict[str, int] = {}
for c in reals:
if (getattr(c, "status", "") or "").lower() == "running":
nid = getattr(c, "node_id", None)
if nid:
real_running_by_node[nid] = real_running_by_node.get(nid, 0) + 1
ghosts = []
for p in _placement(svc) if svc is not None else []:
nid = p.get("node_id", "")
placed = int(p.get("running", 0) or 0)
ghost = placed - real_running_by_node.get(nid, 0)
if ghost > 0:
ghosts.append({
"ghost": True,
"host": node_hostname.get(nid) or (nid[:12] if nid else "?"),
"count": ghost,
})
ghosts.sort(key=lambda g: g["host"].lower())
replicas.extend(ghosts)
running = getattr(svc, "running", None) if svc is not None else None
desired = getattr(svc, "desired", None) if svc is not None else None
# No service row (manager didn't report it): infer from what we can see.
if running is None:
running = sum(1 for r in replicas
if (not r["ghost"] and (r["status"] or "").lower() == "running"))
if desired is None:
desired = len(replicas)
out_services.append({
"name": name,
"mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
"running": running,
"desired": desired,
"state": _service_state(int(running or 0), int(desired or 0)),
"replicas": replicas,
})
# Non-swarm containers grouped by host, running-first order preserved.
groups: dict[str, dict] = {}
for c in standalone_cs:
hid = getattr(c, "host_id", None)
g = groups.get(hid)
if g is None:
g = groups[hid] = {"host_id": hid, "host": host_name.get(hid) or hid or "?",
"containers": []}
g["containers"].append(c)
standalone = sorted(groups.values(), key=lambda g: g["host"].lower())
return {"services": out_services, "standalone": standalone}
@@ -0,0 +1,17 @@
{# docker/_container_history.html — CPU/mem sparklines for the selected range #}
{% if have_data %}
<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:1.5rem;">
<div>
<div class="section-title" style="margin-bottom:0.4rem;">CPU %</div>
{{ sparkline_cpu | safe }}
</div>
<div>
<div class="section-title" style="margin-bottom:0.4rem;">Memory %</div>
{{ sparkline_mem | safe }}
</div>
</div>
{% else %}
<div style="color:var(--text-muted);font-size:0.85rem;">
Not enough samples in this range yet — history fills in as the agent reports.
</div>
{% endif %}
@@ -0,0 +1,110 @@
{# docker/container_detail.html — full detail page for one container #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs %}
{% block title %}{{ name }} — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), (name, "")]) }}{% endblock %}
{% block content %}
{% if container is none %}
<div class="card" style="text-align:center;padding:3rem;">
<div style="font-weight:600;margin-bottom:0.4rem;">Container not found</div>
<div style="color:var(--text-muted);font-size:0.9rem;">
No container named <code>{{ name }}</code> is currently reported for this host.
It may have been removed, or the host agent hasn't reported recently.
</div>
</div>
{% else %}
{# ── Header ──────────────────────────────────────────────────────────────── #}
<div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.35rem;flex-wrap:wrap;">
<span class="dot {% if container.status == 'running' %}dot-up{% elif container.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<h1 class="page-title" style="margin-bottom:0;">{{ container.name }}</h1>
{% if container.health == 'healthy' %}<span style="font-size:0.72rem;color:var(--green);border:1px solid var(--green);border-radius:3px;padding:0.05rem 0.4rem;">healthy</span>
{% elif container.health == 'unhealthy' %}<span style="font-size:0.72rem;color:var(--red);border:1px solid var(--red);border-radius:3px;padding:0.05rem 0.4rem;">unhealthy</span>
{% elif container.health == 'starting' %}<span style="font-size:0.72rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0.05rem 0.4rem;">starting</span>{% endif %}
</div>
<div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
{{ container.status }}{% if uptime %} · up {{ uptime }}{% endif %}
{% if host %} · on <a href="/hosts/{{ host.id }}" style="color:var(--text-muted);">{{ host.name }}</a>{% endif %}
</div>
{# ── Facts grid ──────────────────────────────────────────────────────────── #}
{% macro fact(label, value, colour="") %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">{{ label }}</div>
<div style="font-size:0.95rem;{% if colour %}color:{{ colour }};{% endif %}font-family:ui-monospace,monospace;">{{ value }}</div>
</div>
{% endmacro %}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
{{ fact("CPU", "%.1f%%" | format(container.cpu_pct) if container.cpu_pct is not none else "—") }}
{{ fact("Memory", "%.1f%%" | format(container.mem_pct) if container.mem_pct is not none else "—") }}
{{ fact("Restarts", container.restart_count, "var(--orange)" if container.restart_count else "") }}
{{ fact("Last exit", (container.exit_code ~ (" (OOM)" if container.oom_killed else "")) if container.exit_code is not none else "—",
"var(--red)" if (container.exit_code is not none and container.exit_code != 0) else "") }}
{{ fact("Net in", net_rx) }}
{{ fact("Net out", net_tx) }}
{{ fact("Block read", blk_read) }}
{{ fact("Block write", blk_write) }}
</div>
{# ── Image / placement ───────────────────────────────────────────────────── #}
<div class="card" style="margin-bottom:1.5rem;">
<div style="display:grid;grid-template-columns:max-content 1fr;gap:0.4rem 1.25rem;font-size:0.86rem;">
<span style="color:var(--text-muted);">Image</span>
<span style="font-family:ui-monospace,monospace;word-break:break-all;">{{ container.image or "—" }}</span>
{% if container.compose_project %}
<span style="color:var(--text-muted);">Compose project</span><span>{{ container.compose_project }}</span>
{% endif %}
{% if container.service_name %}
<span style="color:var(--text-muted);">Swarm service</span><span>{{ container.service_name }}</span>
{% endif %}
{% if container.node_id %}
<span style="color:var(--text-muted);">Node</span><span style="font-family:ui-monospace,monospace;">{{ container.node_id }}</span>
{% endif %}
{% if ports %}
<span style="color:var(--text-muted);">Ports</span>
<span style="font-family:ui-monospace,monospace;">{% for p in ports %}{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}{% if not loop.last %}, {% endif %}{% endfor %}</span>
{% endif %}
</div>
</div>
{# ── Resource history (range-toggled) ────────────────────────────────────── #}
<div class="card" style="margin-bottom:1.5rem;">
<div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
<h3 class="section-title" style="margin-bottom:0;">Resource history</h3>
{% include "_time_range.html" %}
</div>
<div id="container-history"
hx-get="/plugins/docker/container/{{ host_id }}/{{ name }}/history"
hx-trigger="load, rangeChange from:body"
hx-include="#time-range"
hx-swap="innerHTML">
<div style="color:var(--text-muted);font-size:0.85rem;">Loading…</div>
</div>
</div>
{# ── Lifecycle timeline ──────────────────────────────────────────────────── #}
<div class="card">
<h3 class="section-title" style="margin-bottom:0.75rem;">Lifecycle</h3>
{% if events %}
<div style="display:grid;gap:0.5rem;">
{% for e in events %}
<div style="display:flex;align-items:baseline;gap:0.6rem;font-size:0.85rem;">
<span style="color:{{ e.colour }};width:1rem;text-align:center;flex-shrink:0;">{{ e.glyph }}</span>
<span style="font-weight:500;width:6.5rem;flex-shrink:0;">{{ e.event }}</span>
<span style="color:var(--text-muted);flex:1;min-width:0;">{{ e.detail or "" }}</span>
<span style="color:var(--text-dim);font-size:0.78rem;white-space:nowrap;" title="{{ e.at }}">{{ e.at.strftime("%Y-%m-%d %H:%M") }}</span>
</div>
{% endfor %}
</div>
{% else %}
<div style="color:var(--text-muted);font-size:0.85rem;">
No lifecycle events recorded yet. Start/stop/health changes appear here as the
agent reports them over time.
</div>
{% endif %}
</div>
{% endif %}
{% endblock %}
+83
View File
@@ -0,0 +1,83 @@
{# docker/disk.html — image/disk usage: reclaimable space, per-image sizes #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs %}
{% block title %}Disk — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Image & disk usage", "")]) }}{% endblock %}
{% block content %}
<h1 class="page-title" style="margin-bottom:0.4rem;">Image &amp; disk usage</h1>
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.5rem;">
Reclaimable = space held by images no container references. Cleanup actions
(prune) arrive in a later release — these are read-only figures for now.
</p>
{% if host_groups %}
{% for g in host_groups %}
<div style="margin-bottom:2rem;">
<div style="font-weight:600;font-size:0.95rem;margin-bottom:0.75rem;">{{ g.host_name }}</div>
{# ── Summary stats ────────────────────────────────────────────────────── #}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1rem;">
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Reclaimable</div>
<span class="stat-val" style="color:{% if g.summary.images_reclaimable != '0 B' %}var(--orange){% else %}var(--green){% endif %};">{{ g.summary.images_reclaimable }}</span>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Images</div>
<span class="stat-val">{{ g.summary.images_size }}</span>
<div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.images_active }}/{{ g.summary.images_total }} in use</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Stopped</div>
<span class="stat-val" style="{% if g.stopped %}color:var(--text-muted){% endif %};">{{ g.stopped }}</span>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Writable layers</div>
<span class="stat-val">{{ g.summary.containers_size }}</span>
<div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.containers_count }} containers</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Volumes</div>
<span class="stat-val">{{ g.summary.volumes_size }}</span>
<div style="font-size:0.74rem;color:var(--text-muted);">{{ g.summary.volumes_count }} volumes</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.3rem;">Build cache</div>
<span class="stat-val">{{ g.summary.build_cache_size }}</span>
</div>
</div>
{# ── Per-image table ──────────────────────────────────────────────────── #}
<div class="card-flush">
<table class="table">
<thead>
<tr><th>Image</th><th>Size</th><th>Shared</th><th>Containers</th></tr>
</thead>
<tbody>
{% for im in g.images %}
<tr>
<td style="font-size:0.84rem;font-family:ui-monospace,monospace;max-width:340px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
{{ im.repo_tag }}
{% if im.reclaimable %}<span title="not referenced by any container" style="font-size:0.68rem;color:var(--orange);border:1px solid var(--orange);border-radius:3px;padding:0 0.3rem;margin-left:0.4rem;">reclaimable</span>{% endif %}
</td>
<td style="font-family:ui-monospace,monospace;font-size:0.86rem;">{{ im.size }}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.86rem;color:var(--text-muted);">{{ im.shared }}</td>
<td style="font-size:0.86rem;color:{% if im.containers %}var(--text){% else %}var(--text-muted){% endif %};">{{ im.containers }}</td>
</tr>
{% else %}
<tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No images reported.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{% endfor %}
{% else %}
<div class="card" style="text-align:center;padding:3rem;">
<div style="color:var(--text-muted);font-size:0.9rem;">
No disk usage reported yet. The host agent reports image/disk usage from
<code>docker system df</code> on hosts running Docker.
</div>
</div>
{% endif %}
{% endblock %}
@@ -0,0 +1,22 @@
{# docker/host_panel.html — per-host Docker fragment embedded on the Hosts hub #}
<div class="card">
<div style="display:flex;align-items:baseline;justify-content:space-between;gap:0.6rem;margin-bottom:0.6rem;">
<h3 class="section-title" style="margin-bottom:0;">Docker</h3>
<span style="font-size:0.78rem;color:var(--text-muted);">
{{ running }} running{% if stopped %} · {{ stopped }} stopped{% endif %}
<a href="/plugins/docker/" style="margin-left:0.6rem;color:var(--text-muted);">All →</a>
</span>
</div>
<div style="display:grid;gap:2px;">
{% for c in containers %}
<div style="display:flex;align-items:center;gap:0.5rem;font-size:0.85rem;padding:0.2rem 0;">
<span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;" title="{{ c.image }}">{{ c.name }}</a>
<span style="font-size:0.74rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
{% if c.cpu_pct is not none %}{{ "%.1f" | format(c.cpu_pct) }}%{% endif %}
{% if c.mem_pct is not none %} · {{ "%.1f" | format(c.mem_pct) }}%{% endif %}
</span>
</div>
{% endfor %}
</div>
</div>
@@ -0,0 +1,24 @@
{% extends "base.html" %}
{% block title %}Docker — Steward{% endblock %}
{% block content %}
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1.5rem;gap:1rem;flex-wrap:wrap;">
<div style="display:flex;align-items:baseline;gap:1rem;">
<h1 class="page-title" style="margin-bottom:0;">Docker</h1>
{% if has_swarm %}
<a href="/plugins/docker/swarm" style="font-size:0.85rem;color:var(--accent);text-decoration:none;">Swarm →</a>
{% endif %}
{% if has_disk %}
<a href="/plugins/docker/disk" style="font-size:0.85rem;color:var(--accent);text-decoration:none;">Disk →</a>
{% endif %}
</div>
{% include "_time_range.html" %}
</div>
<div id="docker-rows"
hx-get="/plugins/docker/rows"
hx-trigger="load, rangeChange from:body"
hx-include="#time-range"
hx-swap="innerHTML">
<div style="color:var(--text-muted);font-size:0.9rem;padding:2rem;">Loading...</div>
</div>
{% endblock %}
+174
View File
@@ -0,0 +1,174 @@
{# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #}
{# ── Summary strip ─────────────────────────────────────────────────────────── #}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(140px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Running</div>
<span class="stat-val" style="color:var(--green);">{{ running }}</span>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Stopped</div>
<span class="stat-val" style="{% if stopped %}color:var(--text-muted){% endif %};">{{ stopped }}</span>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Total</div>
<span class="stat-val">{{ total }}</span>
</div>
{% if swarm_services %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Services</div>
<span class="stat-val">{{ swarm_services | length }}</span>
</div>
{% endif %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Hosts</div>
<span class="stat-val">{{ host_groups | length }}</span>
</div>
</div>
{# ── Swarm services: collapsed per-service, each replica labelled with its host.
Ghost replicas (tasks on nodes with no Steward agent) come from the managers'
placement data so the picture is cluster-complete. ──────────────────────── #}
{% if swarm_services %}
<div class="card-flush" style="margin-bottom:1.5rem;">
<div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
<span style="font-weight:600;font-size:0.95rem;">Swarm services</span>
<a href="/plugins/docker/swarm" style="font-size:0.78rem;color:var(--text-muted);">Topology →</a>
</div>
<div style="padding:0.5rem 0.75rem;display:grid;gap:0.85rem;">
{% for s in swarm_services %}
<div>
<div style="display:flex;align-items:center;gap:0.55rem;flex-wrap:wrap;margin-bottom:0.35rem;">
<span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
<span style="font-weight:600;font-size:0.9rem;">{{ s.name }}</span>
<span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
<span style="font-size:0.78rem;color:{% if s.state == 'healthy' %}var(--green){% elif s.state == 'degraded' %}var(--orange){% else %}var(--red){% endif %};font-variant-numeric:tabular-nums;">
{{ s.running }}/{{ s.desired }} running
</span>
</div>
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(210px,1fr));gap:0.4rem;padding-left:1.1rem;">
{% for r in s.replicas %}
{% if r.ghost %}
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:5px;padding:0.3rem 0.55rem;" title="Running on a node with no Steward agent — detail unavailable">
<span class="dot dot-warn" style="opacity:0.6;"></span>
<span style="font-weight:500;">{{ r.host }}</span>
<span style="color:var(--text-dim);margin-left:auto;">{{ r.count }} · no agent</span>
</div>
{% else %}
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.8rem;background:var(--bg-elevated);border:1px solid var(--border);border-radius:5px;padding:0.3rem 0.55rem;">
<span class="dot {% if r.status == 'running' %}dot-up{% elif r.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<a href="/plugins/docker/container/{{ r.host_id }}/{{ r.name }}" style="font-weight:500;color:inherit;text-decoration:none;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" title="{{ r.name }}">{{ r.host }}</a>
{% if r.cpu_pct is not none %}<span style="color:var(--text-muted);font-family:ui-monospace,monospace;margin-left:auto;">{{ "%.0f" | format(r.cpu_pct) }}%</span>{% endif %}
</div>
{% endif %}
{% endfor %}
</div>
</div>
{% endfor %}
</div>
</div>
{% endif %}
{# ── Container tables, one per host ───────────────────────────────────────── #}
{% if host_groups %}
{% for g in host_groups %}
<div class="card-flush" style="margin-bottom:1.5rem;">
<div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
{% if g.host %}
<a href="/hosts/{{ g.host.id }}" style="font-weight:600;font-size:0.95rem;color:inherit;text-decoration:none;">{{ g.host.name }}</a>
{% else %}
<span style="font-weight:600;font-size:0.95rem;">{{ g.host_name }}</span>
{% endif %}
<span style="font-size:0.78rem;color:var(--text-muted);">
{{ g.running }} running{% if g.stopped %} · {{ g.stopped }} stopped{% endif %}
</span>
</div>
<table class="table">
<thead>
<tr>
<th>Container</th>
<th>Image</th>
<th>Ports</th>
<th>CPU %</th>
<th style="min-width:100px;">CPU history</th>
<th>Mem %</th>
<th style="min-width:100px;">Mem history</th>
</tr>
</thead>
<tbody>
{% for sub in g.subgroups %}
{% if g.grouped and sub.label %}
<tr><td colspan="7" style="padding:0.4rem 0.75rem 0.2rem;font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;font-weight:600;">{{ sub.label }}</td></tr>
{% endif %}
{% for item in sub.containers %}
{% set c = item.container %}
<tr>
<td>
<div style="display:flex;align-items:center;gap:0.5rem;">
<span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<div>
<div style="font-weight:500;font-size:0.9rem;">
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="color:inherit;text-decoration:none;border-bottom:1px dotted var(--border-mid);">{{ c.name }}</a>
{% if c.health == 'healthy' %}<span title="healthy" style="color:var(--green);font-size:0.7rem;"></span>
{% elif c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);font-size:0.7rem;"></span>
{% elif c.health == 'starting' %}<span title="health: starting" style="color:var(--orange);font-size:0.7rem;"></span>{% endif %}
</div>
<div style="font-size:0.73rem;color:var(--text-muted);">
{{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
{% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
· <span style="color:var(--red);">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
{% endif %}
{% if c.restart_count %} · <span title="restart count" style="color:var(--orange);">⟳{{ c.restart_count }}</span>{% endif %}
</div>
{% if c.service_name or c.compose_project %}
<div style="font-size:0.68rem;color:var(--text-dim);margin-top:0.1rem;">
{{ c.service_name or c.compose_project }}
</div>
{% endif %}
</div>
</div>
</td>
<td style="font-size:0.82rem;color:var(--text-muted);max-width:200px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
{{ c.image }}
</td>
<td style="font-size:0.78rem;font-family:ui-monospace,monospace;">
{% for p in item.ports %}
<div style="color:var(--text-muted);">{{ p.host_port }}:{{ p.container_port }}/{{ p.protocol }}</div>
{% endfor %}
</td>
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
{% if c.cpu_pct is not none %}
<span style="color:{% if c.cpu_pct > 80 %}var(--red){% elif c.cpu_pct > 50 %}var(--orange){% else %}var(--text){% endif %};">
{{ "%.1f" | format(c.cpu_pct) }}%
</span>
{% else %}
<span style="color:var(--text-dim);"></span>
{% endif %}
</td>
<td>{{ item.sparkline_cpu | safe }}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
{% if c.mem_pct is not none %}
<span style="color:{% if c.mem_pct > 90 %}var(--red){% elif c.mem_pct > 70 %}var(--orange){% else %}var(--text){% endif %};">
{{ "%.1f" | format(c.mem_pct) }}%
</span>
{% else %}
<span style="color:var(--text-dim);"></span>
{% endif %}
</td>
<td>{{ item.sparkline_mem | safe }}</td>
</tr>
{% endfor %}
{% endfor %}
</tbody>
</table>
</div>
{% endfor %}
{% elif not swarm_services %}
<div class="card" style="text-align:center;padding:3rem;">
<div style="color:var(--text-muted);font-size:0.9rem;">
No containers reported yet. Containers are collected by the Steward host agent —
deploy the agent to a host running Docker (Hosts → the host → agent panel) and
its containers will appear here under that host.
</div>
</div>
{% endif %}
@@ -0,0 +1,87 @@
{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs %}
{% block title %}Swarm — Docker — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Swarm", "")]) }}{% endblock %}
{% block content %}
<h1 class="page-title" style="margin-bottom:1.5rem;">Swarm</h1>
{% if swarms %}
{% for g in swarms %}
<div style="margin-bottom:2rem;">
<div style="display:flex;align-items:baseline;gap:0.6rem;margin-bottom:0.75rem;flex-wrap:wrap;">
<span style="font-weight:600;font-size:0.95rem;">Swarm</span>
<span style="font-size:0.78rem;color:var(--text-muted);">
reported by {{ g.managers | length }} manager{{ 's' if g.managers | length != 1 }} · {{ g.managers | join(", ") }}
</span>
</div>
{# ── Services ─────────────────────────────────────────────────────────── #}
<div class="card-flush" style="margin-bottom:1rem;">
<table class="table">
<thead>
<tr>
<th>Service</th><th>Mode</th><th>Replicas</th><th>Image</th><th>Placement</th>
</tr>
</thead>
<tbody>
{% for s in g.services %}
<tr>
<td style="font-weight:500;font-size:0.9rem;">{{ s.name }}</td>
<td style="font-size:0.82rem;color:var(--text-muted);">{{ s.mode }}</td>
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
<span style="color:{% if s.healthy %}var(--green){% elif s.down %}var(--red){% elif s.degraded %}var(--orange){% else %}var(--text-muted){% endif %};">
{{ s.running }}/{{ s.desired }}
</span>
</td>
<td style="font-size:0.8rem;color:var(--text-muted);max-width:220px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.image or "—" }}</td>
<td style="font-size:0.78rem;color:var(--text-muted);">
{% for p in s.placement %}{{ p.node }} ({{ p.running }}){% if not loop.last %}, {% endif %}{% else %}—{% endfor %}
</td>
</tr>
{% else %}
<tr><td colspan="5" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No services in this swarm.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
{# ── Nodes ────────────────────────────────────────────────────────────── #}
<div class="card-flush">
<table class="table">
<thead>
<tr><th>Node</th><th>Role</th><th>Availability</th><th>Status</th></tr>
</thead>
<tbody>
{% for n in g.nodes %}
<tr>
<td style="font-weight:500;font-size:0.9rem;">
{{ n.hostname or n.node_id }}
{% if n.leader %}<span title="cluster leader" style="font-size:0.68rem;color:var(--accent);border:1px solid var(--accent);border-radius:3px;padding:0 0.3rem;margin-left:0.3rem;">leader</span>{% endif %}
</td>
<td style="font-size:0.82rem;color:var(--text-muted);">{{ n.role }}</td>
<td style="font-size:0.82rem;color:{% if n.availability == 'active' %}var(--text){% else %}var(--orange){% endif %};">{{ n.availability }}</td>
<td style="font-size:0.82rem;">
<span class="dot {% if n.status == 'ready' %}dot-up{% else %}dot-down{% endif %}"></span>
<span style="color:{% if n.status == 'ready' %}var(--green){% else %}var(--red){% endif %};">{{ n.status }}</span>
</td>
</tr>
{% else %}
<tr><td colspan="4" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No nodes reported.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{% endfor %}
{% else %}
<div class="card" style="text-align:center;padding:3rem;">
<div style="color:var(--text-muted);font-size:0.9rem;">
No Swarm topology reported. A Steward host agent running on a Swarm
<strong>manager</strong> reports services, nodes, and task placement here —
workers and non-swarm hosts contribute only their local containers.
</div>
</div>
{% endif %}
{% endblock %}
@@ -0,0 +1,67 @@
{# docker/widget.html — dashboard widget: container status overview, by host #}
{% if total_count == 0 %}
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No containers reported yet.</p>
{% else %}
<div style="display:flex;gap:1.25rem;margin-bottom:0.65rem;">
<div>
<span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--green);">{{ running_count }}</span>
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">running</span>
</div>
<div title="Distinct containers with a die or OOM event in the last 24h">
<span style="font-size:1.4rem;font-weight:700;line-height:1;color:{% if failed_24h %}var(--red){% else %}var(--text-muted){% endif %};">{{ failed_24h }}</span>
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">failed (24h)</span>
</div>
</div>
{# Swarm services — collapsed, each with its replicas' host chips (dashed = a
node with no Steward agent, count-only from placement). #}
{% if swarm_services %}
<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">Swarm services</div>
<div style="display:grid;gap:0.4rem;margin-bottom:0.4rem;">
{% for s in swarm_services %}
<div>
<div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
<span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
<span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
<span style="color:var(--text-muted);font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;">{{ s.running }}/{{ s.desired }}</span>
</div>
<div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">
{% for r in s.replicas %}
{% if r.ghost %}
<span title="no Steward agent on this node — count from swarm placement" style="font-size:0.68rem;color:var(--text-dim);background:var(--bg-elevated);border:1px dashed var(--border-mid);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }} ×{{ r.count }}</span>
{% else %}
<span style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:4px;padding:0.05rem 0.35rem;">{{ r.host }}</span>
{% endif %}
{% endfor %}
</div>
</div>
{% endfor %}
</div>
{% endif %}
{% for g in host_groups %}
{% if multi_host %}
<div style="font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;margin:0.5rem 0 0.2rem;">{{ g.host_name }}</div>
{% endif %}
<div style="display:grid;gap:2px;">
{% for c in g.containers %}
<div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
<span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;">
{{ c.name }}{% if c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);"></span>{% elif c.health == 'healthy' %}<span title="healthy" style="color:var(--green);"></span>{% endif %}{% if c.restart_count %}<span title="restarts" style="color:var(--orange);font-size:0.7rem;"> ⟳{{ c.restart_count }}</span>{% endif %}
</a>
{% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
<span title="last exit code" style="font-size:0.7rem;color:var(--red);flex-shrink:0;">exit {{ c.exit_code }}</span>
{% endif %}
{% if c.cpu_pct is not none %}
<span style="font-size:0.72rem;color:var(--text-muted);font-family:ui-monospace,monospace;flex-shrink:0;">
{{ "%.1f" | format(c.cpu_pct) }}%
</span>
{% endif %}
</div>
{% endfor %}
</div>
{% endfor %}
{% endif %}
@@ -0,0 +1,29 @@
{# docker/widget_resources.html — dashboard widget: the busiest running containers
by CPU, each with labeled CPU + memory utilisation bars. #}
{% if not rows %}
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No running containers.</div>
{% else %}
{% macro bar(label, pct, warn, crit) %}
<div style="display:flex;align-items:center;gap:0.45rem;margin-bottom:2px;">
<span style="font-size:0.62rem;color:var(--text-dim);width:1.9rem;flex-shrink:0;letter-spacing:0.03em;">{{ label }}</span>
<div style="flex:1;height:6px;background:var(--bg-elevated);border-radius:3px;overflow:hidden;">
<div style="height:100%;border-radius:3px;width:{{ [pct, 100] | min }}%;
background:{% if pct > crit %}var(--red){% elif pct > warn %}var(--orange){% else %}var(--accent){% endif %};
transition:width 0.4s;"></div>
</div>
<span style="font-size:0.7rem;color:var(--text-muted);font-family:ui-monospace,monospace;width:3rem;text-align:right;flex-shrink:0;">{{ "%.1f" | format(pct) }}%</span>
</div>
{% endmacro %}
<div style="display:grid;gap:0.6rem;">
{% for r in rows %}
{% set c = r.c %}
<div>
<div style="font-size:0.8rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-bottom:0.25rem;">
{{ c.name }}{% if multi_host %}<span style="color:var(--text-dim);font-weight:normal;"> · {{ r.host_name }}</span>{% endif %}
</div>
{% if c.cpu_pct is not none %}{{ bar("CPU", c.cpu_pct, 50, 80) }}{% endif %}
{% if c.mem_pct is not none %}{{ bar("MEM", c.mem_pct, 70, 90) }}{% endif %}
</div>
{% endfor %}
</div>
{% endif %}
+23
View File
@@ -0,0 +1,23 @@
# plugins/host_agent/__init__.py
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from quart import Quart
_app: "Quart | None" = None
def setup(app: "Quart") -> None:
global _app
_app = app
def get_scheduled_tasks() -> list:
from .scheduler import make_task
return [make_task(_app)]
def get_blueprint():
from .routes import host_agent_bp
return host_agent_bp
File diff suppressed because it is too large Load Diff
+112
View File
@@ -0,0 +1,112 @@
"""Host-metric read helpers (latest snapshot + bucketed history).
Kept separate from routes.py so it imports only the core PluginMetric model — not
the host_agent ORM models — which lets integration tests import these helpers
without tripping the plugin-loader's double-registration ("Table already
defined") guard.
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from sqlalchemy import func, or_, select
from steward.core.time_range import bucket_seconds
from steward.models.metrics import PluginMetric, PluginMetricHourly
SOURCE_MODULE = "host_agent"
# Host-level metrics charted on the detail page (sub-resources are shown as
# current-value lists, not time series, to keep the page readable).
HISTORY_METRICS = (
"cpu_pct", "mem_used_pct", "disk_used_pct_worst", "load_1m",
"net_rx_bps", "net_tx_bps", "disk_read_bps", "disk_write_bps",
"temp_c_max", "psi_mem_some_avg10",
)
async def _latest_metrics_for_host(session, host_name: str) -> dict[str, dict[str, float]]:
"""{resource_name: {metric: value}} — latest sample per (resource, metric) for
a host + its sub-resources.
DISTINCT ON picks the newest row per group in one index-ordered pass over
ix_plugin_metrics_module_resource_metric_recorded, instead of a GROUP-BY-max
subquery self-joined back to the table (two passes over the whole history).
"""
rows = (await session.execute(
select(PluginMetric)
.where(
PluginMetric.source_module == SOURCE_MODULE,
or_(
PluginMetric.resource_name == host_name,
PluginMetric.resource_name.like(host_name + ":%"),
),
)
.distinct(PluginMetric.resource_name, PluginMetric.metric_name)
.order_by(
PluginMetric.resource_name,
PluginMetric.metric_name,
PluginMetric.recorded_at.desc(),
)
)).scalars().all()
out: dict[str, dict[str, float]] = {}
for r in rows:
out.setdefault(r.resource_name, {})[r.metric_name] = r.value
return out
async def _history_for_host(session, host_name: str, since, *, raw_days: int = 7) -> dict[str, list]:
"""{metric: [[epoch_ms, avg_value], …]} host-level series since `since`.
Retention rolls raw plugin_metrics older than `raw_days` into hourly averages
(plugin_metrics_hourly) and deletes the raw rows. So we read the rollup for the
part of the range older than that boundary and raw (bucket-averaged in SQL,
capped to ≤1h to match the rollup) for the recent part — never shipping raw
samples to Python. The two windows don't overlap, so appending hourly-then-raw
keeps each metric's series time-ordered. Epoch-ms x feeds a linear chart axis.
"""
now = datetime.now(timezone.utc)
raw_cutoff = (now - timedelta(days=raw_days)).replace(minute=0, second=0, microsecond=0)
series: dict[str, list] = {m: [] for m in HISTORY_METRICS}
# ── Older than the raw window: the hourly rollup ──
if since < raw_cutoff:
hourly = (await session.execute(
select(PluginMetricHourly.metric_name, PluginMetricHourly.bucket,
PluginMetricHourly.value_avg)
.where(
PluginMetricHourly.source_module == SOURCE_MODULE,
PluginMetricHourly.resource_name == host_name,
PluginMetricHourly.metric_name.in_(HISTORY_METRICS),
PluginMetricHourly.bucket >= since,
PluginMetricHourly.bucket < raw_cutoff,
)
.order_by(PluginMetricHourly.bucket)
)).all()
for metric_name, bucket, avg in hourly:
series[metric_name].append([int(bucket.timestamp() * 1000), round(float(avg), 2)])
# ── Recent part: raw, bucket-averaged in SQL ──
raw_since = since if since >= raw_cutoff else raw_cutoff
width_s = bucket_seconds(raw_since, 120)
if since < raw_cutoff:
width_s = min(width_s, 3600) # ≤ 1h so it matches the hourly portion
rbucket = func.date_bin(
func.make_interval(0, 0, 0, 0, 0, 0, width_s),
PluginMetric.recorded_at,
func.to_timestamp(0), # epoch origin
).label("bucket")
raw = (await session.execute(
select(PluginMetric.metric_name, rbucket, func.avg(PluginMetric.value))
.where(
PluginMetric.source_module == SOURCE_MODULE,
PluginMetric.resource_name == host_name,
PluginMetric.metric_name.in_(HISTORY_METRICS),
PluginMetric.recorded_at >= raw_since,
)
.group_by(PluginMetric.metric_name, rbucket)
.order_by(rbucket)
)).all()
for metric_name, b, avg in raw:
series[metric_name].append([int(b.timestamp() * 1000), round(float(avg), 2)])
return series
+70
View File
@@ -0,0 +1,70 @@
# plugins/host_agent/migrations/env.py
"""Alembic env.py for the host_agent plugin."""
import asyncio
import os
import sys
from pathlib import Path
from logging.config import fileConfig
from sqlalchemy import pool
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import async_engine_from_config
from alembic import context
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
from steward.models.base import Base
import steward.models # noqa: F401
from plugins.host_agent.models import HostAgentRegistration # noqa: F401
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def _get_url() -> str:
import yaml
cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml")
try:
with open(cfg_path) as f:
cfg = yaml.safe_load(f) or {}
url = cfg.get("database", {}).get("url", "")
except FileNotFoundError:
url = ""
return os.environ.get("STEWARD_DATABASE__URL", url)
def run_migrations_offline() -> None:
context.configure(
url=_get_url(), target_metadata=target_metadata,
literal_binds=True, dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def do_run_migrations(connection: Connection) -> None:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
cfg = config.get_section(config.config_ini_section, {})
cfg["sqlalchemy.url"] = _get_url()
connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
asyncio.run(run_async_migrations())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
@@ -0,0 +1,36 @@
# plugins/host_agent/migrations/versions/host_agent_001_initial.py
"""host_agent plugin initial tables
Revision ID: host_agent_001_initial
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "host_agent_001_initial"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = "host_agent"
depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
def upgrade() -> None:
op.create_table(
"host_agent_registrations",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("host_id", sa.String(36),
sa.ForeignKey("hosts.id", ondelete="CASCADE"),
nullable=False, unique=True),
sa.Column("token_hash", sa.String(64), nullable=False, unique=True, index=True),
sa.Column("token_created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("agent_version", sa.String(32), nullable=True),
sa.Column("kernel", sa.String(128), nullable=True),
sa.Column("distro", sa.String(128), nullable=True),
sa.Column("arch", sa.String(32), nullable=True),
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
op.drop_table("host_agent_registrations")
@@ -0,0 +1,24 @@
# plugins/host_agent/migrations/versions/host_agent_002_host_ip.py
"""host_agent: add agent-reported host_ip column
Revision ID: host_agent_002_host_ip
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "host_agent_002_host_ip"
down_revision: Union[str, None] = "host_agent_001_initial"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"host_agent_registrations",
sa.Column("host_ip", sa.String(45), nullable=True),
)
def downgrade() -> None:
op.drop_column("host_agent_registrations", "host_ip")
+34
View File
@@ -0,0 +1,34 @@
# plugins/host_agent/models.py
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from sqlalchemy import Column, String, DateTime, ForeignKey
from steward.models.base import Base
def _uuid() -> str:
return str(uuid.uuid4())
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
class HostAgentRegistration(Base):
__tablename__ = "host_agent_registrations"
id = Column(String(36), primary_key=True, default=_uuid)
host_id = Column(String(36), ForeignKey("hosts.id", ondelete="CASCADE"),
unique=True, nullable=False)
token_hash = Column(String(64), nullable=False, unique=True, index=True)
token_created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
agent_version = Column(String(32), nullable=True)
kernel = Column(String(128), nullable=True)
distro = Column(String(128), nullable=True)
arch = Column(String(32), nullable=True)
# Agent-reported primary IP. 45 chars = max textual IPv6 (no zone suffix).
host_ip = Column(String(45), nullable=True)
last_seen_at = Column(DateTime(timezone=True), nullable=True)
created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
updated_at = Column(DateTime(timezone=True), nullable=False,
default=_utcnow, onupdate=_utcnow)
+19
View File
@@ -0,0 +1,19 @@
# plugins/host_agent/plugin.yaml
name: host_agent
version: "1.2.0"
description: "Remote Linux host resource monitoring via a lightweight Python push agent (CPU incl. per-core, memory + PSI, storage, disk I/O, network throughput, load, temperatures, uptime)"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/host_agent"
tags:
- host
- monitoring
- cpu
- memory
- storage
config:
stale_after_seconds: 180
default_interval_seconds: 30
File diff suppressed because it is too large Load Diff
+78
View File
@@ -0,0 +1,78 @@
# plugins/host_agent/scheduler.py
from __future__ import annotations
import logging
from datetime import datetime, timedelta, timezone
from typing import Iterable
from sqlalchemy import select
from steward.core.scheduler import ScheduledTask
from steward.models.hosts import Host
from .models import HostAgentRegistration
logger = logging.getLogger(__name__)
def _filter_stale(
regs: Iterable,
*,
now: datetime,
stale_after_seconds: int,
) -> list:
"""Pure staleness filter: returns the subset with last_seen_at strictly
older than (now - stale_after_seconds). Rows with last_seen_at=None are
never stale (they are unregistered-in-practice)."""
cutoff = now - timedelta(seconds=stale_after_seconds)
return [
r for r in regs
if r.last_seen_at is not None and r.last_seen_at < cutoff
]
async def find_stale_registrations(app, stale_after_seconds: int = 180) -> list[dict]:
async with app.db_sessionmaker() as session:
all_regs = (await session.execute(
select(HostAgentRegistration)
)).scalars().all()
stale_rows = _filter_stale(
all_regs,
now=datetime.now(timezone.utc),
stale_after_seconds=stale_after_seconds,
)
if not stale_rows:
return []
hosts = {
h.id: h for h in (await session.execute(
select(Host).where(Host.id.in_([r.host_id for r in stale_rows]))
)).scalars().all()
}
return [
{
"host_id": r.host_id,
"host_name": hosts[r.host_id].name if r.host_id in hosts else "?",
"last_seen_at": r.last_seen_at,
}
for r in stale_rows
]
def make_task(app) -> ScheduledTask:
async def _check_stale():
try:
stale = await find_stale_registrations(app)
except Exception:
logger.exception("host_agent stale check failed")
return
if stale:
logger.info(
"host_agent: %d stale agent(s): %s",
len(stale),
[s["host_name"] for s in stale],
)
return ScheduledTask(
name="host_agent_stale_check",
coro_factory=_check_stale,
interval_seconds=60,
run_on_startup=False,
)
@@ -0,0 +1,6 @@
{# History data-only fragment. Swapped into the hidden #hm-charts-data poller;
the script runs and feeds the persistent charts created by host_detail.html,
updating them in place (no canvas swap → no flicker / no re-animation). #}
<script>
if (window.applyHostSeries) window.applyHostSeries({{ series|tojson }}, {{ range_key|tojson }});
</script>
@@ -0,0 +1,182 @@
{# Current-state fragment for the full-metrics page — polled live via HTMX.
Self-contained (defines its own format macros) so each poll re-renders the
latest snapshot the server has. #}
{% macro fmt_bps(v) %}
{%- if v is none -%}—
{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s
{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s
{%- else -%}{{ "%.0f"|format(v) }} B/s
{%- endif -%}
{% endmacro %}
{% macro fmt_bytes(v) %}
{%- if v is none -%}—
{%- elif v >= 1125899906842624 -%}{{ "%.1f"|format(v / 1125899906842624) }} PB
{%- elif v >= 1099511627776 -%}{{ "%.1f"|format(v / 1099511627776) }} TB
{%- elif v >= 1073741824 -%}{{ "%.1f"|format(v / 1073741824) }} GB
{%- elif v >= 1048576 -%}{{ "%.0f"|format(v / 1048576) }} MB
{%- else -%}{{ "%.0f"|format(v / 1024) }} KB
{%- endif -%}
{% endmacro %}
{% macro bar(pct) %}
{%- set p = pct if pct is not none else 0 -%}
<div style="height:8px;background:var(--bg-elevated);border-radius:4px;overflow:hidden;border:1px solid var(--border);">
<div style="height:100%;width:{{ p|round(1) }}%;background:{% if pct is none %}var(--text-dim){% elif pct < 70 %}var(--green){% elif pct < 90 %}var(--yellow){% else %}var(--red){% endif %};"></div>
</div>
{% endmacro %}
{# ── Identity / metadata ─────────────────────────────────────────────────── #}
<div class="card" style="display:flex;flex-wrap:wrap;gap:0.5rem 1.25rem;align-items:center;font-size:0.82rem;color:var(--text-muted);">
<span>{% if stale %}<span class="badge badge-red">stale</span>{% else %}<span class="badge badge-green">live</span>{% endif %}</span>
<span><span style="color:var(--text-dim);">Address</span> {{ host.address or "—" }}</span>
{% if reg %}
{% if reg.distro %}<span><span style="color:var(--text-dim);">OS</span> {{ reg.distro }}</span>{% endif %}
{% if reg.kernel %}<span><span style="color:var(--text-dim);">Kernel</span> {{ reg.kernel }}</span>{% endif %}
{% if reg.arch %}<span><span style="color:var(--text-dim);">Arch</span> {{ reg.arch }}</span>{% endif %}
{% if reg.agent_version %}<span><span style="color:var(--text-dim);">Agent</span> v{{ reg.agent_version }}</span>{% endif %}
{% set up = hostlvl.get('uptime_secs') %}
{% if up %}<span><span style="color:var(--text-dim);">Uptime</span> {{ (up // 86400)|int }}d {{ ((up % 86400) // 3600)|int }}h</span>{% endif %}
<span><span style="color:var(--text-dim);">Last seen</span> {{ reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") ~ " UTC" if reg.last_seen_at else "never" }}</span>
{% else %}
<span>No agent registration found for this host.</span>
{% endif %}
</div>
{% if not hostlvl %}
<div class="card empty">No metrics reported yet. Once the agent checks in, CPU, memory, network, disk I/O, temperatures, and pressure will appear here.</div>
{% else %}
{# ── Current headline gauges ─────────────────────────────────────────────── #}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.75rem;margin-bottom:1rem;">
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">CPU</div>
{% set c = hostlvl.get('cpu_pct') %}
<span class="stat-val" style="font-size:1.6rem;color:{% if c is none %}var(--text-dim){% elif c < 70 %}var(--green){% elif c < 90 %}var(--yellow){% else %}var(--red){% endif %};">{% if c is not none %}{{ "%.0f"|format(c) }}%{% else %}—{% endif %}</span>
<div style="margin-top:0.5rem;">{{ bar(c) }}</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Memory</div>
{% set mp = hostlvl.get('mem_used_pct') %}
<span class="stat-val" style="font-size:1.6rem;color:{% if mp is none %}var(--text-dim){% elif mp < 70 %}var(--green){% elif mp < 90 %}var(--yellow){% else %}var(--red){% endif %};">{% if mp is not none %}{{ "%.0f"|format(mp) }}%{% else %}—{% endif %}</span>
<div style="margin-top:0.5rem;">{{ bar(mp) }}</div>
<div style="margin-top:0.45rem;font-size:0.72rem;color:var(--text-dim);">
avail {{ fmt_bytes(hostlvl.get('mem_available_bytes')) }} · cache {{ fmt_bytes(hostlvl.get('mem_cached_bytes')) }} · swap {{ fmt_bytes(hostlvl.get('swap_used_bytes')) }}
</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Load</div>
<span class="stat-val" style="font-size:1.6rem;">{% if hostlvl.get('load_1m') is not none %}{{ "%.2f"|format(hostlvl.get('load_1m')) }}{% else %}—{% endif %}</span>
<div style="margin-top:0.45rem;font-size:0.72rem;color:var(--text-dim);">
5m {% if hostlvl.get('load_5m') is not none %}{{ "%.2f"|format(hostlvl.get('load_5m')) }}{% else %}—{% endif %} ·
15m {% if hostlvl.get('load_15m') is not none %}{{ "%.2f"|format(hostlvl.get('load_15m')) }}{% else %}—{% endif %}
</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Network</div>
<div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
<div><span style="color:var(--green);"></span> {{ fmt_bps(hostlvl.get('net_rx_bps')) }}</div>
<div><span style="color:var(--red);"></span> {{ fmt_bps(hostlvl.get('net_tx_bps')) }}</div>
</div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Disk I/O</div>
<div style="font-size:0.95rem;font-variant-numeric:tabular-nums;">
<div><span style="color:var(--text-dim);">rd</span> {{ fmt_bps(hostlvl.get('disk_read_bps')) }}</div>
<div><span style="color:var(--text-dim);">wr</span> {{ fmt_bps(hostlvl.get('disk_write_bps')) }}</div>
</div>
</div>
{% if hostlvl.get('temp_c_max') is not none %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;">Temp (max)</div>
{% set t = hostlvl.get('temp_c_max') %}
<span class="stat-val" style="font-size:1.6rem;color:{% if t < 70 %}var(--green){% elif t < 85 %}var(--yellow){% else %}var(--red){% endif %};">{{ "%.0f"|format(t) }}°C</span>
</div>
{% endif %}
{% if hostlvl.get('psi_mem_some_avg10') is not none %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.35rem;" title="Pressure stall — % of time stalled, 10s avg">Pressure (10s)</div>
<div style="font-size:0.78rem;font-variant-numeric:tabular-nums;color:var(--text-muted);">
<div>mem {{ "%.1f"|format(hostlvl.get('psi_mem_some_avg10')) }}%</div>
{% if hostlvl.get('psi_cpu_some_avg10') is not none %}<div>cpu {{ "%.1f"|format(hostlvl.get('psi_cpu_some_avg10')) }}%</div>{% endif %}
{% if hostlvl.get('psi_io_some_avg10') is not none %}<div>io {{ "%.1f"|format(hostlvl.get('psi_io_some_avg10')) }}%</div>{% endif %}
</div>
</div>
{% endif %}
</div>
{# ── Per-core CPU ─────────────────────────────────────────────────────────── #}
{% if cores %}
<div class="card">
<div class="section-title" style="margin-bottom:0.6rem;">Per-core CPU</div>
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(120px,1fr));gap:0.5rem 0.9rem;">
{% for idx, pct in cores %}
<div>
<div style="display:flex;justify-content:space-between;font-size:0.72rem;color:var(--text-dim);margin-bottom:0.2rem;">
<span>core {{ idx }}</span><span>{% if pct is not none %}{{ "%.0f"|format(pct) }}%{% else %}—{% endif %}</span>
</div>
{{ bar(pct) }}
</div>
{% endfor %}
</div>
</div>
{% endif %}
{# ── Filesystems ──────────────────────────────────────────────────────────── #}
{% if mounts %}
<div class="card">
<div class="section-title" style="margin-bottom:0.6rem;">Filesystems</div>
{% for mount, m in mounts.items() %}
<div style="margin-bottom:0.6rem;">
<div style="display:flex;justify-content:space-between;font-size:0.8rem;margin-bottom:0.2rem;">
<span style="font-family:ui-monospace,monospace;">{{ mount }}</span>
<span style="color:var(--text-muted);">{{ fmt_bytes(m.get('disk_used_bytes')) }} / {{ fmt_bytes(m.get('disk_total_bytes')) }}{% if m.get('disk_used_pct') is not none %} · {{ "%.0f"|format(m.get('disk_used_pct')) }}%{% endif %}</span>
</div>
{{ bar(m.get('disk_used_pct')) }}
</div>
{% endfor %}
</div>
{% endif %}
{# ── Interfaces / disks (short panels, side by side at natural height) ─────── #}
{% if nets or disks_io %}
<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(260px,1fr));gap:1rem;align-items:start;">
{% if nets %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.5rem;">Interfaces</div>
{% for iface, m in nets.items() %}
<div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
<span style="font-family:ui-monospace,monospace;">{{ iface }}</span>
<span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">↓ {{ fmt_bps(m.get('net_rx_bps')) }} · ↑ {{ fmt_bps(m.get('net_tx_bps')) }}</span>
</div>
{% endfor %}
</div>
{% endif %}
{% if disks_io %}
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.5rem;">Disks</div>
{% for dev, m in disks_io.items() %}
<div style="display:flex;justify-content:space-between;gap:0.75rem;font-size:0.8rem;padding:0.15rem 0;">
<span style="font-family:ui-monospace,monospace;">{{ dev }}</span>
<span style="color:var(--text-muted);font-variant-numeric:tabular-nums;white-space:nowrap;">rd {{ fmt_bps(m.get('disk_read_bps')) }} · wr {{ fmt_bps(m.get('disk_write_bps')) }}</span>
</div>
{% endfor %}
</div>
{% endif %}
</div>
{% endif %}
{# ── Temperatures (full width; cores flow into a compact multi-column grid) ── #}
{% if temps %}
<div class="card" style="margin-top:1rem;margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.5rem;">Temperatures</div>
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(130px,1fr));gap:0.1rem 1.25rem;">
{% for label, c in temps.items() %}
<div style="display:flex;justify-content:space-between;gap:0.5rem;font-size:0.8rem;padding:0.15rem 0;">
<span style="white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ label }}</span>
<span style="color:{% if c is none %}var(--text-dim){% elif c < 70 %}var(--green){% elif c < 85 %}var(--yellow){% else %}var(--red){% endif %};font-variant-numeric:tabular-nums;">{% if c is not none %}{{ "%.0f"|format(c) }}°C{% else %}—{% endif %}</span>
</div>
{% endfor %}
</div>
</div>
{% endif %}
{% endif %}
@@ -0,0 +1,35 @@
{# Host vitals strip — compact CPU/MEM/DISK/LOAD + pressure, live-polled into the
host hub. Renders nothing until the agent reports (the Agent panel below then
shows provisioning). #}
{% if reporting %}
{% set lbl = "font-size:0.68rem;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em;" %}
{% macro vital(label, value, text, kind, spark) %}
<div style="min-width:88px;">
<div style="{{ lbl }}">{{ label }}</div>
<div style="font-size:1.5rem;font-weight:700;line-height:1.1;{{ threshold_style(value, kind) }}">{{ text }}</div>
<div style="margin-top:0.2rem;">{{ spark | safe }}</div>
</div>
{% endmacro %}
<div class="card" style="display:flex;flex-wrap:wrap;align-items:flex-start;gap:1rem 2rem;margin-bottom:1rem;">
{{ vital("CPU", cpu, ('%.0f%%'|format(cpu) if cpu is not none else '—'), 'cpu', sparks.cpu) }}
{{ vital("Memory", mem, ('%.0f%%'|format(mem) if mem is not none else '—'), 'mem', sparks.mem) }}
{{ vital("Disk /", disk_root, ('%.0f%%'|format(disk_root) if disk_root is not none else '—'), 'disk', sparks.disk) }}
{{ vital("Load /core", load_per_core, ('%d%%'|format(load_per_core) if load_per_core is not none else '—'), 'load', sparks.load) }}
<div style="margin-left:auto;display:flex;flex-direction:column;align-items:flex-end;gap:0.3rem;font-size:0.75rem;color:var(--text-muted);text-align:right;">
<span>
{% if stale %}<span class="badge badge-red">stale</span>{% else %}<span class="badge badge-green">live</span>{% endif %}
{% if reg and reg.agent_version %}<span style="margin-left:0.3rem;">v{{ reg.agent_version }}</span>{% endif %}
</span>
{% if psi.cpu is not none or psi.mem is not none or psi.io is not none %}
<span title="Pressure stall — % of the last 10s tasks waited for the resource">
<span style="color:var(--text-dim);text-transform:uppercase;font-size:0.66rem;letter-spacing:0.04em;">Pressure 10s</span>
cpu {{ '%.0f%%'|format(psi.cpu) if psi.cpu is not none else '—' }}
· mem {{ '%.0f%%'|format(psi.mem) if psi.mem is not none else '—' }}
· io {{ '%.0f%%'|format(psi.io) if psi.io is not none else '—' }}
</span>
{% endif %}
{% if reg and reg.last_seen_at %}<span>seen {{ reg.last_seen_at.strftime("%H:%M:%S") }} UTC</span>{% endif %}
</div>
</div>
{% endif %}
@@ -0,0 +1,108 @@
{% extends "base.html" %}
{% from "_macros.html" import crumbs %}
{% block title %}{{ host.name }} — Metrics — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), (host.name, "/hosts/" ~ host.id), ("Metrics", "")]) }}{% endblock %}
{% block content %}
{# Shell only: current-state + history data stream in via HTMX so the heavy
history query never blocks first paint and the data refreshes live (≈ the
agent's push cadence). The chart CANVASES live here permanently — only their
data is polled and applied in place, so refreshes never re-create/flicker. #}
<div style="display:flex;align-items:center;justify-content:space-between;gap:1rem;flex-wrap:wrap;margin-bottom:0.75rem;">
<h1 class="page-title" style="margin-bottom:0;">{{ host.name }}</h1>
{% include "_time_range.html" %}
</div>
{# Current state — lazy + polled live. Atomic innerHTML swap of fixed-height
cards, so values update without a layout jump. #}
<div id="hm-current"
hx-get="/plugins/host_agent/{{ host.id }}/metrics"
hx-trigger="load, every {{ poll_seconds }}s"
hx-swap="innerHTML">
<div class="card empty">Loading metrics…</div>
</div>
{# History charts — persistent canvases (created once below); only data is polled. #}
<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(340px,1fr));gap:1rem;margin-top:1rem;">
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.5rem;">Utilization % — last <span class="hm-chart-range">{{ current_range }}</span></div>
<div style="height:220px;"><canvas id="chart-util"></canvas></div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.5rem;">Throughput (B/s) — last <span class="hm-chart-range">{{ current_range }}</span></div>
<div style="height:220px;"><canvas id="chart-net"></canvas></div>
</div>
<div class="card" style="margin-bottom:0;">
<div class="section-title" style="margin-bottom:0.5rem;">Load &amp; pressure — last <span class="hm-chart-range">{{ current_range }}</span></div>
<div style="height:220px;"><canvas id="chart-pressure"></canvas></div>
</div>
</div>
{# Hidden poller: fetches the history series and applies it to the charts above
(no visible swap). Lazy on load → never blocks paint; refreshed slowly + on
range change. #}
<div id="hm-charts-data" style="display:none;"
hx-get="/plugins/host_agent/{{ host.id }}/charts"
hx-trigger="load, every {{ chart_poll_seconds }}s, rangeChange from:body"
hx-include="#time-range"
hx-swap="innerHTML"></div>
<script>
(function () {
const fmtTime = (v) => { const d = new Date(v); return ("0" + d.getHours()).slice(-2) + ":" + ("0" + d.getMinutes()).slice(-2); };
const baseOpts = (ymax) => ({
responsive: true, maintainAspectRatio: false,
animation: false, // no grow-in / re-animate on refresh
interaction: { mode: "index", intersect: false },
elements: { point: { radius: 0 } },
scales: {
x: { type: "linear", ticks: { callback: (v) => fmtTime(v), maxTicksLimit: 8, color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
y: { beginAtZero: true, max: ymax, ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
},
plugins: {
legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } },
tooltip: { callbacks: { title: (items) => items.length ? fmtTime(items[0].parsed.x) : "" } },
},
});
const defs = {
"chart-util": { ymax: 100, ds: [
{ key: "cpu_pct", label: "CPU %", color: "#c8a840" },
{ key: "mem_used_pct", label: "Mem %", color: "#4aa86a" },
{ key: "disk_used_pct_worst", label: "Disk % (worst)", color: "#c87840" },
] },
"chart-net": { ymax: undefined, ds: [
{ key: "net_rx_bps", label: "Net RX", color: "#4aa86a" },
{ key: "net_tx_bps", label: "Net TX", color: "#c84048" },
{ key: "disk_read_bps", label: "Disk read", color: "#6aa0d0" },
{ key: "disk_write_bps", label: "Disk write", color: "#c8a840" },
] },
"chart-pressure": { ymax: undefined, ds: [
{ key: "load_1m", label: "Load 1m", color: "#c8a840" },
{ key: "psi_mem_some_avg10", label: "Mem PSI %", color: "#c84048" },
{ key: "temp_c_max", label: "Temp °C", color: "#c87840" },
] },
};
const charts = {};
for (const id of Object.keys(defs)) {
const el = document.getElementById(id);
if (!el) continue;
charts[id] = new Chart(el.getContext("2d"), {
type: "line",
data: { datasets: defs[id].ds.map((d) => ({ label: d.label, data: [], borderColor: d.color, backgroundColor: d.color, borderWidth: 1.5, tension: 0.25 })) },
options: baseOpts(defs[id].ymax),
});
}
// Called by the polled /charts data fragment. Updates data in place (no
// re-create, no animation), so refreshes never flicker or shift layout.
window.applyHostSeries = function (series, rangeKey) {
for (const id of Object.keys(defs)) {
const ch = charts[id];
if (!ch) continue;
defs[id].ds.forEach((d, i) => {
ch.data.datasets[i].data = (series[d.key] || []).map((p) => ({ x: p[0], y: p[1] }));
});
ch.update("none");
}
if (rangeKey) document.querySelectorAll(".hm-chart-range").forEach((s) => { s.textContent = rangeKey; });
};
})();
</script>
{% endblock %}
@@ -0,0 +1,81 @@
#!/bin/sh
# Steward host agent installer
# Generated for: {{ host_name }} ({{ host_address }})
# Steward URL: {{ url }}
set -e
STEWARD_URL="{{ url }}"
AGENT_TOKEN="{{ token }}"
AGENT_VERSION="{{ agent_version }}"
AGENT_USER="steward-agent"
AGENT_DIR="/usr/local/lib/steward-agent"
CONF_FILE="/etc/steward-agent.conf"
UNIT_FILE="/etc/systemd/system/steward-agent.service"
# ── preflight ────────────────────────────────────────────────────────────────
[ "$(id -u)" = "0" ] || { echo "must run as root (use sudo)"; exit 1; }
command -v systemctl >/dev/null 2>&1 || { echo "systemd not found — unsupported init system"; exit 1; }
command -v python3 >/dev/null 2>&1 || { echo "python3 not found — install python3 first"; exit 1; }
# Handle --uninstall
if [ "${1:-}" = "--uninstall" ]; then
systemctl disable --now steward-agent.service 2>/dev/null || true
rm -f "$UNIT_FILE" "$CONF_FILE"
rm -rf "$AGENT_DIR"
systemctl daemon-reload
# keep the system user — removing it risks orphaned files elsewhere
echo "uninstalled"
exit 0
fi
# ── create system user ───────────────────────────────────────────────────────
if ! id "$AGENT_USER" >/dev/null 2>&1; then
useradd --system --no-create-home --shell /usr/sbin/nologin "$AGENT_USER"
fi
# ── drop the agent file ──────────────────────────────────────────────────────
mkdir -p "$AGENT_DIR"
curl -sSL "${STEWARD_URL}/plugins/host_agent/agent.py" -o "$AGENT_DIR/agent.py"
chmod 0755 "$AGENT_DIR/agent.py"
chown root:root "$AGENT_DIR/agent.py"
# ── write config ─────────────────────────────────────────────────────────────
cat > "$CONF_FILE" <<EOF
url = $STEWARD_URL
token = $AGENT_TOKEN
interval_seconds = 30
EOF
chown "root:$AGENT_USER" "$CONF_FILE"
chmod 0640 "$CONF_FILE"
# ── write systemd unit ───────────────────────────────────────────────────────
cat > "$UNIT_FILE" <<EOF
[Unit]
Description=Steward host agent
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=$AGENT_USER
ExecStart=/usr/bin/python3 $AGENT_DIR/agent.py
Restart=on-failure
RestartSec=10
NoNewPrivileges=yes
ProtectSystem=strict
ProtectHome=yes
PrivateTmp=yes
ReadOnlyPaths=/proc /sys
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable --now steward-agent.service
echo
echo "Steward host agent $AGENT_VERSION installed and running."
echo "Check status: systemctl status steward-agent"
echo "Logs: journalctl -u steward-agent -f"
+146
View File
@@ -0,0 +1,146 @@
{# Per-host agent panel — embedded into /hosts/<id> via HTMX. Self-contained. #}
{% set scope = "steward:target:" ~ target.id if target else "" %}
<div class="card">
<div style="display:flex;align-items:baseline;justify-content:space-between;margin-bottom:0.75rem;gap:1rem;flex-wrap:wrap;">
<h3 class="section-title" style="margin-bottom:0;">Agent</h3>
{% if reporting %}
<span style="font-size:0.78rem;color:{{ 'var(--yellow)' if stale else 'var(--green)' }};">
{{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %}
· last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }}
</span>
{% elif reg %}
<span style="font-size:0.78rem;color:var(--text-muted);">pending — no check-in yet</span>
{% endif %}
</div>
{% if reporting %}
{# Reporting: live vitals are shown by the vitals strip above; this panel is
lifecycle/management only. #}
<div style="display:flex;gap:0.5rem;flex-wrap:wrap;align-items:center;">
<a href="/plugins/host_agent/{{ host.id }}/" class="btn btn-sm btn-ghost">Full metrics →</a>
{% if session.user_role == 'admin' %}
{% if ansible_available and target %}
<form method="post" action="/plugins/host_agent/update" style="margin:0;">
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<button type="submit" class="btn btn-sm" title="Refresh agent.py + restart (token preserved)">Update agent</button>
</form>
{% endif %}
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/rotate-token" style="margin:0;"
onsubmit="return confirm('Rotate token? The agent stops reporting until reinstalled/updated with the new token.');">
<button type="submit" class="btn btn-sm btn-ghost">Rotate token</button>
</form>
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0;"
onsubmit="return confirm('Remove this agent registration? Metrics stop until re-registered.');">
<button type="submit" class="btn btn-sm btn-danger">Remove</button>
</form>
{% endif %}
</div>
{% if ansible_available and not target %}
<p style="color:var(--text-dim);font-size:0.8rem;margin:0.6rem 0 0;">
Link an Ansible target (above) to enable one-click agent updates.
</p>
{% endif %}
{% if session.user_role == 'admin' and ansible_available and target and managed_key_set %}
<details style="margin-top:0.7rem;">
<summary style="cursor:pointer;font-size:0.8rem;color:var(--text-muted);">Re-provision (reinstall the steward account + managed key, then the agent)</summary>
<p style="font-size:0.78rem;color:var(--text-dim);margin:0.5rem 0;">
Use after regenerating the managed key, or if SSH auth as <code>steward</code> breaks.
Connects over a one-time bootstrap user + password (not stored).
</p>
<form method="post" action="/plugins/host_agent/provision"
style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;">
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<div class="form-group" style="margin:0;">
<label>Bootstrap user</label>
<input type="text" name="bootstrap_user" required placeholder="root" style="width:8rem;" autocomplete="off">
</div>
<div class="form-group" style="margin:0;">
<label>Bootstrap password</label>
<input type="password" name="bootstrap_password" required style="width:10rem;" autocomplete="new-password">
</div>
<button type="submit" class="btn btn-sm">Re-provision</button>
</form>
</details>
{% endif %}
{% else %}
{# ── Not reporting: pending (token minted, no check-in) or never installed ── #}
{% if reg %}
<div style="background:color-mix(in srgb,var(--yellow) 10%,var(--bg-elevated));
border:1px solid color-mix(in srgb,var(--yellow) 30%,var(--border));
border-radius:6px;padding:0.7rem 0.9rem;margin-bottom:0.75rem;font-size:0.84rem;">
A token was minted for this host but <strong>no metrics have arrived yet</strong> — the
deploy may still be running, or it failed. Open the latest
<a href="/ansible/">Ansible run</a> to check, then retry below. Live metrics appear here
once the agent checks in.
</div>
{% else %}
<p style="color:var(--text-muted);font-size:0.88rem;margin-bottom:0.75rem;">
No agent installed. The agent reports CPU, memory, disk, network and more back to Steward.
</p>
{% endif %}
{% if session.user_role != 'admin' %}
<p style="color:var(--text-dim);font-size:0.85rem;">An admin can install the agent here.</p>
{% elif not ansible_available %}
<p style="color:var(--text-dim);font-size:0.85rem;">
Ansible isn't available, so the agent can't be deployed from here. Install it manually with the
<a href="/plugins/host_agent/fleet/">curl install command</a>.
</p>
{% elif not managed_key_set %}
<div style="background:color-mix(in srgb,var(--yellow) 12%,var(--bg-elevated));
border:1px solid color-mix(in srgb,var(--yellow) 35%,var(--border));
border-radius:6px;padding:0.7rem 0.9rem;display:flex;align-items:center;gap:0.8rem;flex-wrap:wrap;">
<span style="color:var(--yellow);"></span>
<span style="font-size:0.84rem;flex:1;min-width:13rem;">No managed SSH key yet — Steward needs one to log into hosts.</span>
<form method="post" action="/settings/ansible/generate-key" style="margin:0;">
<input type="hidden" name="next" value="/hosts/{{ host.id }}">
<button type="submit" class="btn btn-sm">Generate managed key</button>
</form>
</div>
{% elif not target %}
<p style="color:var(--text-dim);font-size:0.85rem;">
Link or create an Ansible target for this host (in the Ansible section below) first — provisioning
needs an SSH connection.
</p>
{% else %}
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.75rem;">
Deploys the agent to <code>{{ target.name }}</code> by running an Ansible playbook. Steward mints
the host's API token automatically; you'll watch the run live.
</p>
{# Provision: brand-new host (creates steward account + key over a one-time password) #}
<form method="post" action="/plugins/host_agent/provision"
style="display:flex;gap:0.6rem;align-items:flex-end;flex-wrap:wrap;margin-bottom:0.75rem;">
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<div style="font-size:0.78rem;color:var(--text-muted);width:100%;">Provision (first contact — fresh host):</div>
<div class="form-group" style="margin:0;">
<label>Bootstrap user</label>
<input type="text" name="bootstrap_user" required placeholder="root" style="width:8rem;" autocomplete="off">
</div>
<div class="form-group" style="margin:0;">
<label>Bootstrap password</label>
<input type="password" name="bootstrap_password" required style="width:10rem;" autocomplete="new-password">
</div>
<button type="submit" class="btn btn-sm">Provision</button>
</form>
{# Install: host already has the steward account (managed key works) #}
<form method="post" action="/plugins/host_agent/deploy" style="display:flex;gap:0.6rem;align-items:center;">
<input type="hidden" name="inventory_scope" value="{{ scope }}">
<span style="font-size:0.78rem;color:var(--text-muted);">Already provisioned?</span>
<button type="submit" class="btn btn-sm btn-ghost">Install agent (managed key)</button>
</form>
{% endif %}
{% if reg and session.user_role == 'admin' %}
<form method="post" action="/plugins/host_agent/fleet/{{ host.id }}/delete" style="margin:0.75rem 0 0;"
onsubmit="return confirm('Clear this pending registration? Its token is discarded.');">
<button type="submit" class="btn btn-sm btn-ghost">Clear pending registration</button>
</form>
{% endif %}
{% endif %}
</div>
@@ -0,0 +1,175 @@
{% extends "base.html" %}
{% from "_macros.html" import crumbs %}
{% block title %}Agent fleet — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), ("Agent fleet", "")]) }}{% endblock %}
{% block content %}
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1rem;gap:1rem;flex-wrap:wrap;">
<h1 class="page-title" style="margin-bottom:0;">Agent fleet</h1>
</div>
<p style="color:var(--text-muted);font-size:0.84rem;margin:-0.4rem 0 1.25rem;">
Bulk agent operations across your inventory. For a single host, use its
<a href="/hosts/">host page</a> — provisioning and metrics live there.
</p>
{% if new_token %}
<div class="card" style="background:var(--accent-bg);">
<h3>New token — copy this install command now</h3>
<p style="font-size:0.82rem;color:var(--text-muted);">
This is the only time the raw token is shown. Rotate if you lose it.
</p>
<pre style="user-select:all;word-break:break-all;">curl -sSL '{{ install_url }}' | sudo sh</pre>
</div>
{% endif %}
<div class="card">
<form method="post" action="/plugins/host_agent/fleet/add-host"
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
<div class="form-group" style="margin-bottom:0;">
<label>Host name</label>
<input type="text" name="name" required>
</div>
<div class="form-group" style="margin-bottom:0;">
<label>Address (optional)</label>
<input type="text" name="address">
</div>
<button type="submit" class="btn">Add host</button>
</form>
</div>
{% if ansible_available %}
{% set scope_select %}
<label>Target</label>
<select name="inventory_scope" required>
{% for g in deploy_groups %}<option value="steward:group:{{ g.id }}">Group: {{ g.name }}</option>{% endfor %}
{% for t in deploy_targets %}<option value="steward:target:{{ t.id }}">Target: {{ t.name }}</option>{% endfor %}
</select>
{% endset %}
<div class="card">
<h3 style="margin-bottom:0.4rem;">Agent lifecycle via Ansible</h3>
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.6rem;">
These actions run bundled <strong>Ansible playbooks</strong> to deploy and maintain the
Steward monitoring agent on your <a href="/ansible/inventory/targets">inventory hosts</a>.
Steward generates each host's API token automatically and connects over SSH as the managed
<code>steward</code> account — you'll be dropped into the live Ansible run to watch it.
</p>
{% if not managed_key_set %}
<div style="background:color-mix(in srgb,var(--yellow) 12%,var(--bg-elevated));
border:1px solid color-mix(in srgb,var(--yellow) 35%,var(--border));
border-radius:6px;padding:0.7rem 0.9rem;display:flex;align-items:center;
gap:0.8rem;flex-wrap:wrap;">
<span style="color:var(--yellow);"></span>
<span style="font-size:0.84rem;flex:1;min-width:14rem;">
No managed SSH key yet — Steward needs one to log into hosts as <code>steward</code>.
</span>
<form method="post" action="/settings/ansible/generate-key" style="margin:0;">
<input type="hidden" name="next" value="/plugins/host_agent/fleet/">
<button type="submit" class="btn btn-sm">Generate managed key</button>
</form>
</div>
{% elif not (deploy_targets or deploy_groups) %}
<p style="font-size:0.85rem;color:var(--text-dim);margin:0;">
No Ansible inventory targets yet. Add some under <a href="/ansible/inventory/targets">Ansible → Inventory</a>.
</p>
{% endif %}
</div>
{% if managed_key_set and (deploy_targets or deploy_groups) %}
<div class="card">
<h3 style="margin-bottom:0.4rem;">1 · Provision a fresh host</h3>
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.75rem;">
First contact for a brand-new host. Connects with a one-time <strong>bootstrap</strong>
user + password (used for this run only, never stored), creates the <code>steward</code>
login account with passwordless sudo, installs the managed SSH key, then installs the
agent. Run this once per host.
</p>
<form method="post" action="/plugins/host_agent/provision"
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
<div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
<div class="form-group" style="margin-bottom:0;">
<label>Bootstrap user</label>
<input type="text" name="bootstrap_user" required placeholder="root" style="width:9rem;" autocomplete="off">
</div>
<div class="form-group" style="margin-bottom:0;">
<label>Bootstrap password</label>
<input type="password" name="bootstrap_password" required style="width:11rem;" autocomplete="new-password">
</div>
<div class="form-group" style="margin-bottom:0;">
<label>Report interval (s)</label>
<input type="number" name="agent_interval" value="30" min="5" style="width:7rem;">
</div>
<button type="submit" class="btn">Provision</button>
</form>
</div>
<div class="card">
<h3 style="margin-bottom:0.4rem;">2 · Install / enroll agent</h3>
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.75rem;">
For a host that's already provisioned (has the <code>steward</code> account) but has no
agent yet — or to re-enroll one. Connects as <code>steward</code> with the managed key,
mints a fresh token, and installs the agent. No <code>curl | sh</code> needed.
</p>
<form method="post" action="/plugins/host_agent/deploy"
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
<div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
<div class="form-group" style="margin-bottom:0;">
<label>Report interval (s)</label>
<input type="number" name="agent_interval" value="30" min="5" style="width:7rem;">
</div>
<button type="submit" class="btn">Install</button>
</form>
</div>
<div class="card">
<h3 style="margin-bottom:0.4rem;">3 · Update agents</h3>
<p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.75rem;">
Refresh the agent binary on hosts already running it. Connects as <code>steward</code>
with the managed key and restarts the service — the token and config are left untouched,
so identity is preserved.
</p>
<form method="post" action="/plugins/host_agent/update"
style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">
<div class="form-group" style="margin-bottom:0;">{{ scope_select }}</div>
<button type="submit" class="btn">Update</button>
</form>
</div>
{% endif %}
{% endif %}
<div class="card-flush">
<table class="table">
<thead>
<tr>
<th>Host</th><th>Reported IP</th><th>Agent version</th><th>Distro</th>
<th>Last seen</th><th class="td-actions">Actions</th>
</tr>
</thead>
<tbody>
{% for item in registrations %}
<tr>
<td>{{ item.host.name if item.host else item.reg.host_id }}</td>
<td>{{ item.reg.host_ip or "—" }}</td>
<td>{{ item.reg.agent_version or "—" }}</td>
<td>{{ item.reg.distro or "—" }}</td>
<td>{{ item.reg.last_seen_at.strftime("%Y-%m-%d %H:%M:%S") if item.reg.last_seen_at else "never" }}</td>
<td class="td-actions">
<form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/rotate-token"
style="display:inline;"
onsubmit="return confirm('Rotate the token for this host? The existing agent will stop working until reinstalled with the new token.');">
<button type="submit" class="btn btn-ghost btn-sm">Rotate token</button>
</form>
<form method="post" action="/plugins/host_agent/fleet/{{ item.reg.host_id }}/delete"
style="display:inline;"
onsubmit="return confirm('Delete this host registration? The agent will be unable to report metrics until re-registered.');">
<button type="submit" class="btn btn-danger btn-sm">Delete</button>
</form>
</td>
</tr>
{% else %}
<tr><td colspan="6" class="empty">No hosts registered. Add one above.</td></tr>
{% endfor %}
</tbody>
</table>
</div>
{% endblock %}
@@ -0,0 +1,88 @@
{# host_agent history-graph widget — the host-view utilization chart, embedded
on a dashboard. Fills the (resizable) panel; epoch-ms linear axis so no
Chart.js date adapter is needed. #}
{% set has_data = host and (series.cpu_pct or series.mem_used_pct or series.disk_root) %}
{% if not host %}
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.75rem 0;">
No host selected. <strong>Edit</strong> this dashboard and pick a host for this graph.
</div>
{% elif not has_data %}
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.75rem 0;">
No agent metrics for <strong>{{ host.name }}</strong> in the last {{ hours }}h yet.
</div>
{% else %}
<div style="display:flex;flex-direction:column;height:100%;min-height:200px;">
{# Name the host the graph represents (panel title is generic) + a live range
toggle that re-requests this widget without entering edit mode. #}
<div style="flex:0 0 auto;display:flex;align-items:center;gap:0.5rem;flex-wrap:wrap;font-size:0.75rem;color:var(--text-muted);margin-bottom:0.3rem;">
{% if session.user_id %}
<a href="/hosts/{{ host.id }}" style="font-weight:600;">{{ host.name }}</a>
{% else %}
<strong style="color:var(--text);">{{ host.name }}</strong>
{% endif %}
<span class="hist-range" style="margin-left:auto;display:inline-flex;border:1px solid var(--border-mid);border-radius:5px;overflow:hidden;">
{% for opt in [1, 6, 24] %}
<button type="button" onclick="histRange({{ wid }}, '{{ host.id }}', {{ opt }})"
style="border:0;cursor:pointer;padding:0.1rem 0.45rem;font-size:0.72rem;
background:{% if opt == hours %}var(--accent){% else %}transparent{% endif %};
color:{% if opt == hours %}#fff{% else %}var(--text-muted){% endif %};">{{ opt }}h</button>
{% endfor %}
</span>
</div>
<div style="flex:1 1 auto;min-height:150px;position:relative;">
<canvas id="host-hist-{{ wid }}"></canvas>
</div>
</div>
<script>
// Live time-range switch. Rewrites the parent cell's hx-get so the choice sticks
// across polls (htmx re-reads the attribute each poll), then fetches immediately.
window.histRange = function (wid, hostId, hours) {
var cell = document.getElementById("widget-" + wid);
if (!cell) return;
var base = (cell.getAttribute("hx-get") || "").split("?")[0];
var url = base + "?host_id=" + encodeURIComponent(hostId) + "&hours=" + hours + "&wid=" + wid;
cell.setAttribute("hx-get", url);
if (window.htmx) window.htmx.ajax("GET", url, { target: cell, swap: "innerHTML" });
};
(function () {
var el = document.getElementById("host-hist-{{ wid }}");
if (!el) return;
var series = {{ series|tojson }};
var fmtTime = function (v) {
var d = new Date(v);
return ("0" + d.getHours()).slice(-2) + ":" + ("0" + d.getMinutes()).slice(-2);
};
var ds = [
{ key: "cpu_pct", label: "CPU %", color: "#c8a840" },
{ key: "mem_used_pct", label: "Mem %", color: "#4aa86a" },
{ key: "disk_root", label: "Disk / %", color: "#c87840" },
];
new Chart(el.getContext("2d"), {
type: "line",
data: { datasets: ds.map(function (d) {
return {
label: d.label,
data: (series[d.key] || []).map(function (p) { return { x: p[0], y: p[1] }; }),
borderColor: d.color, backgroundColor: d.color, borderWidth: 1.5, tension: 0.25,
};
}) },
options: {
responsive: true, maintainAspectRatio: false,
interaction: { mode: "index", intersect: false },
elements: { point: { radius: 0 } },
scales: {
x: { type: "linear", ticks: { callback: function (v) { return fmtTime(v); }, maxTicksLimit: 6, color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
// Fixed 0{{ y_max }} ceiling (server snaps the data peak up to the next
// 10% band). Steady across refreshes — auto-scaling made the zoom jump on
// every poll — while a banded ceiling still keeps the lines filling the panel.
y: { min: 0, max: {{ y_max }}, ticks: { color: "#8a8a92", font: { size: 10 } }, grid: { color: "#30303a" } },
},
plugins: {
legend: { labels: { color: "#b8b8b0", boxWidth: 10, font: { size: 11 } } },
tooltip: { callbacks: { title: function (items) { return items.length ? fmtTime(items[0].parsed.x) : ""; } } },
},
},
});
})();
</script>
{% endif %}
@@ -0,0 +1,77 @@
{# host_agent widget — fleet glance. One horizontal row per host, zebra-striped
so adjacent hosts read as distinct groupings: health dot + host name on the
left, metric cells (cpu/mem/disk/load) each with a trend sparkline laid out
horizontally on the right. The name wraps (never hard-truncated) but the row
stays horizontal. #}
{% if rows %}
{% macro metric_cell(label, value, fmt, svg, style="", title="") %}
<div style="min-width:62px;flex:0 0 auto;" title="{{ title }}">
<div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
<div style="font-weight:600;font-size:0.8rem;font-variant-numeric:tabular-nums;{{ style }}">
{% if value is not none %}{{ fmt|format(value) }}{% else %}—{% endif %}
</div>
<div style="line-height:0;height:16px;">{{ svg | safe }}</div>
</div>
{% endmacro %}
{# Human-readable throughput, matching the host-detail page. #}
{% macro fmt_bps(v) %}
{%- if v is none -%}—
{%- elif v >= 1048576 -%}{{ "%.1f"|format(v / 1048576) }} MB/s
{%- elif v >= 1024 -%}{{ "%.1f"|format(v / 1024) }} KB/s
{%- else -%}{{ "%.0f"|format(v) }} B/s
{%- endif -%}
{% endmacro %}
{# Two-line down/up (or read/write) throughput cell. #}
{% macro io_cell(label, down_sym, down_val, up_sym, up_val, title="") %}
<div style="min-width:80px;flex:0 0 auto;" title="{{ title }}">
<div style="font-size:0.62rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.03em;">{{ label }}</div>
<div style="font-size:0.72rem;font-variant-numeric:tabular-nums;line-height:1.3;">
<div>{{ down_sym }} {{ fmt_bps(down_val) }}</div>
<div>{{ up_sym }} {{ fmt_bps(up_val) }}</div>
</div>
</div>
{% endmacro %}
<div class="host-fleet">
{% for r in rows %}
{% set stale = r.stale %}
<div style="display:flex;align-items:center;gap:0.6rem 1rem;flex-wrap:wrap;padding:0.5rem 0.6rem;border-radius:4px;{% if loop.index0 % 2 %}background:var(--bg-elevated);{% endif %}">
{# Left — health dot + full host name (wraps, never truncated) + last seen #}
<div style="display:flex;align-items:center;gap:0.5rem;flex:1 1 150px;min-width:120px;">
<span class="dot {% if stale %}dot-dim{% elif (r.cpu_pct or 0) >= 90 or (r.mem_used_pct or 0) >= 90 or (r.disk_worst or 0) >= 90 %}dot-warn{% else %}dot-up{% endif %}"
style="flex-shrink:0;"
title="{% if stale %}stale / no recent data{% else %}warns at CPU/memory ≥90% or any disk mount ≥90% (the 'disk /' figure shows root only){% endif %}"></span>
<span style="min-width:0;">
{% if session.user_id %}
<a href="/hosts/{{ r.host.id }}" style="font-weight:600;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</a>
{% else %}
<span style="font-weight:600;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</span>
{% endif %}
<span style="display:block;font-size:0.7rem;color:var(--text-dim);">
{{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }}
</span>
</span>
</div>
{# Right — metric cells with value + trend, laid out horizontally #}
<div style="display:flex;flex-wrap:wrap;gap:0.4rem 1rem;justify-content:flex-end;">
{{ metric_cell("cpu", r.cpu_pct, "%.0f%%", r.sparks.cpu, threshold_style(r.cpu_pct, 'cpu'), "Average CPU utilization") }}
{{ metric_cell("mem", r.mem_used_pct, "%.0f%%", r.sparks.mem, threshold_style(r.mem_used_pct, 'mem'), "Memory in use") }}
{{ metric_cell("disk /", r.disk_root, "%.0f%%", r.sparks.disk, threshold_style(r.disk_root, 'disk'), "Root filesystem (/) usage") }}
{{ metric_cell("load", r.load_1m, "%.2f", r.sparks.load, "", "Load average (1m)") }}
{# Extra agent metrics — shown only when the host reports them (VMs/containers
often have no temp sensor, etc.) so absent data doesn't clutter the row. #}
{% if r.net_rx_bps is not none or r.net_tx_bps is not none %}
{{ io_cell("net", "↓", r.net_rx_bps, "↑", r.net_tx_bps, "Network throughput (down / up)") }}
{% endif %}
{% if r.disk_read_bps is not none or r.disk_write_bps is not none %}
{{ io_cell("disk i/o", "rd", r.disk_read_bps, "wr", r.disk_write_bps, "Disk I/O (read / write)") }}
{% endif %}
{% if r.temp_max is not none %}
{{ metric_cell("temp", r.temp_max, "%.0f°C", "", threshold_style(r.temp_max, 'temp'), "Hottest sensor") }}
{% endif %}
</div>
</div>
{% endfor %}
</div>
{% else %}
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No hosts with agent data yet.</p>
{% endif %}
+74
View File
@@ -0,0 +1,74 @@
# steward-plugins / index.yaml
#
# Plugin catalog for Steward.
# Fetched by the app at: Settings → Plugins → Plugin Catalog
#
# After updating an entry, commit and push — changes are live within 5 minutes
# (the app caches this file in-process for 300 seconds).
#
# See docs/plugins/index.yaml.example in the main app repo for the full schema.
version: 1
updated: "2026-03-22"
plugins:
- name: docker
version: "2.0.0"
description: "Per-host Docker container status + resource usage, collected by the Steward host agent"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/docker"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/docker-v2.0.0/docker.zip"
checksum_sha256: ""
tags:
- containers
- docker
- infrastructure
- name: traefik
version: "1.0.0"
description: "Traefik reverse proxy metrics and access log integration"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/traefik-v1.0.0/traefik.zip"
checksum_sha256: ""
tags:
- proxy
- metrics
- access-log
- name: unifi
version: "1.0.0"
description: "UniFi Network controller integration — WAN health, devices, clients, DPI"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/unifi-v1.0.0/unifi.zip"
checksum_sha256: ""
tags:
- network
- unifi
- ubiquiti
- name: ups
version: "1.0.0"
description: "UPS monitoring via NUT (Network UPS Tools) with Ansible shutdown automation"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/ups"
download_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins/releases/download/ups-v1.0.0/ups.zip"
checksum_sha256: ""
tags:
- ups
- power
- nut
+28
View File
@@ -0,0 +1,28 @@
# plugins/snmp/__init__.py
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from quart import Quart
_app: "Quart | None" = None
def setup(app: "Quart") -> None:
global _app
_app = app
def get_scheduled_tasks() -> list:
from .scheduler import make_poll_task
return [make_poll_task(_app)]
def get_blueprint():
from .routes import snmp_bp
return snmp_bp
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (SNMP device readings).
return [{"label": "SNMP", "href": "/plugins/snmp/"}]
+59
View File
@@ -0,0 +1,59 @@
# plugins/snmp/plugin.yaml
name: snmp
version: "1.0.0"
description: "SNMP polling for network devices — switches, routers, printers, UPSes, etc."
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/snmp"
tags:
- snmp
- network
- monitoring
config:
poll_interval_seconds: 60
# Each device is polled independently. OIDs must resolve to numeric values.
#
# `host` is the SNMP poll target (IP/hostname we query). A device also shows
# up on a Steward Host's detail page when it maps to that host. By default
# that mapping is implicit: `host` is matched against the Host's address or
# name. To bind explicitly instead — e.g. when you poll by IP but the Host is
# recorded by DNS name, or for an SNMP-only switch/PDU/UPS — add ONE of:
# host_id: "<steward-host-uuid>" # exact Host.id match
# steward_host: "switch01" # Host name OR address (case-insensitive)
# An explicit binding is exclusive: the device then maps ONLY to that host,
# never implicitly to another by a coincidental address string.
devices:
- name: "core-switch"
host: "192.168.1.1"
port: 161
community: "public"
version: "2c" # "1", "2c", or "3"
oids:
- oid: "1.3.6.1.2.1.1.3.0"
label: "uptime_centisecs"
- oid: "1.3.6.1.2.1.2.2.1.10.1"
label: "if1_in_octets"
- oid: "1.3.6.1.2.1.2.2.1.16.1"
label: "if1_out_octets"
# Example: network UPS with SNMP management card (RFC 1628 UPS-MIB).
# upsBatteryStatus values: 1=unknown, 2=batteryNormal, 3=batteryLow, 4=batteryDepleted.
# Uncomment and point at your UPS to enable.
# - name: "ups"
# host: "192.168.1.10"
# port: 161
# community: "public"
# version: "2c"
# oids:
# - oid: "1.3.6.1.2.1.33.1.2.1.0"
# label: "battery_status"
# - oid: "1.3.6.1.2.1.33.1.2.2.0"
# label: "seconds_on_battery"
# - oid: "1.3.6.1.2.1.33.1.2.3.0"
# label: "runtime_minutes_remaining"
# - oid: "1.3.6.1.2.1.33.1.2.4.0"
# label: "battery_charge_pct"
# - oid: "1.3.6.1.2.1.33.1.3.3.1.3.1"
# label: "input_voltage_dV"
+171
View File
@@ -0,0 +1,171 @@
# plugins/snmp/poller.py
"""
Asynchronous SNMP GET helper.
Requires pysnmp-lextudio (the maintained pysnmp fork), bundled into the Docker
image via the `snmp` extra (`pip install .[snmp]`):
pip install 'steward[snmp]'
If pysnmp is not installed, poll_device() returns an empty dict and logs a
warning — SNMP polling is then simply disabled, nothing else breaks.
"""
from __future__ import annotations
import logging
logger = logging.getLogger(__name__)
def _pysnmp_available() -> bool:
try:
import pysnmp # noqa: F401
return True
except ImportError:
return False
def _mp_model(version: str) -> int:
"""Map an SNMP version string to pysnmp's mpModel int (0 = v1, 1 = v2c)."""
return 0 if version == "1" else 1
def _close_engine(engine) -> None:
"""Release the engine's UDP transport socket.
pysnmp opens a UDP socket per ``SnmpEngine`` and never closes it on its own.
Since we build a fresh engine for every poll, an unclosed engine leaks one
file descriptor each scheduler tick; over hours of polling that exhausts the
process fd limit (``OSError: [Errno 24] Too many open files``), which also
takes down the app's listening socket. So close it explicitly here.
The close method/attribute names differ across pysnmp majors (6.2.x lextudio
is camelCase, canonical 7.x is snake_case), so probe for whatever exists.
All paths are best-effort — a failed close must never break the poll loop.
"""
# 7.x exposes a convenience close directly on the engine.
for meth in ("close_dispatcher", "closeDispatcher"):
fn = getattr(engine, meth, None)
if callable(fn):
try:
fn()
except Exception:
pass
return
# 6.2.x: go through the transport dispatcher.
for attr in ("transport_dispatcher", "transportDispatcher"):
dispatcher = getattr(engine, attr, None)
if dispatcher is None:
continue
for meth in ("close_dispatcher", "closeDispatcher"):
fn = getattr(dispatcher, meth, None)
if callable(fn):
try:
fn()
except Exception:
pass
return
return
async def poll_device(
host: str,
port: int,
community: str,
version: str,
oids: list[dict],
) -> dict[str, float]:
"""Perform an SNMP GET for each OID and return ``{label: float_value}``.
Non-numeric OIDs (strings, etc.) are skipped. Returns an empty dict on any
error (unreachable host, wrong community, …) so a flaky device never breaks
the poll loop.
pysnmp's HLAPI is asyncio-only as of v6. The import path moved between major
versions, so we support both rather than let a dependency bump silently
re-break polling:
• pysnmp-lextudio 6.2.x → ``pysnmp.hlapi.asyncio`` (``getCmd`` + a directly
constructed ``UdpTransportTarget``).
• canonical pysnmp 7.x → ``pysnmp.hlapi.v3arch.asyncio`` (``get_cmd`` + the
async ``UdpTransportTarget.create``).
"""
if not _pysnmp_available():
logger.warning("pysnmp not installed — SNMP polling disabled. "
"Install with: pip install 'steward[snmp]'")
return {}
try:
# canonical pysnmp 7.x
from pysnmp.hlapi.v3arch.asyncio import (
CommunityData,
ContextData,
ObjectIdentity,
ObjectType,
SnmpEngine,
UdpTransportTarget,
get_cmd as _get_cmd,
)
_transport_is_async = True
except ImportError:
# pysnmp-lextudio 6.2.x
from pysnmp.hlapi.asyncio import (
CommunityData,
ContextData,
ObjectIdentity,
ObjectType,
SnmpEngine,
UdpTransportTarget,
getCmd as _get_cmd,
)
_transport_is_async = False
engine = SnmpEngine()
# Always release the engine's UDP socket — see _close_engine for why.
try:
# Same host/port for every OID on this device, so build the transport once.
try:
if _transport_is_async:
transport = await UdpTransportTarget.create((host, port), timeout=5, retries=1)
else:
transport = UdpTransportTarget((host, port), timeout=5, retries=1)
except Exception as exc:
logger.debug("SNMP transport setup failed for %s:%s: %s", host, port, exc)
return {}
results: dict[str, float] = {}
for oid_cfg in oids:
oid = oid_cfg["oid"]
label = oid_cfg.get("label") or oid
scale = float(oid_cfg.get("scale", 1.0))
try:
error_indication, error_status, error_index, var_binds = await _get_cmd(
engine,
CommunityData(community, mpModel=_mp_model(version)),
transport,
ContextData(),
ObjectType(ObjectIdentity(oid)),
)
except Exception as exc:
logger.debug("SNMP GET %s@%s OID %s failed: %s", host, port, oid, exc)
continue
if error_indication:
logger.debug("SNMP error %s@%s OID %s: %s", host, port, oid, error_indication)
continue
if error_status:
logger.debug("SNMP status %s@%s OID %s: %s at %s",
host, port, oid, error_status.prettyPrint(),
error_index and var_binds[int(error_index) - 1][0] or "?")
continue
for _, val in var_binds:
try:
results[label] = float(val) * scale
except (TypeError, ValueError):
# Non-numeric type (e.g. OctetString description) — skip.
logger.debug("SNMP non-numeric value for %s label=%s: %r", oid, label, val)
return results
finally:
_close_engine(engine)
+215
View File
@@ -0,0 +1,215 @@
# plugins/snmp/routes.py
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import func, select
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.models.metrics import PluginMetric
snmp_bp = Blueprint("snmp", __name__, template_folder="templates")
async def _latest_readings(db, device_names: list[str]) -> dict[str, dict[str, float]]:
"""Return {device_name: {label: latest_value}} for all configured devices."""
if not device_names:
return {}
# Latest value per (resource_name, metric_name) pair
subq = (
select(
PluginMetric.resource_name,
PluginMetric.metric_name,
func.max(PluginMetric.recorded_at).label("latest_at"),
)
.where(PluginMetric.source_module == "snmp")
.where(PluginMetric.resource_name.in_(device_names))
.group_by(PluginMetric.resource_name, PluginMetric.metric_name)
.subquery()
)
result = await db.execute(
select(PluginMetric).join(
subq,
(PluginMetric.resource_name == subq.c.resource_name)
& (PluginMetric.metric_name == subq.c.metric_name)
& (PluginMetric.recorded_at == subq.c.latest_at),
).where(PluginMetric.source_module == "snmp")
)
rows = result.scalars().all()
out: dict[str, dict[str, float]] = {}
for row in rows:
out.setdefault(row.resource_name, {})[row.metric_name] = row.value
return out
async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]:
"""Return {label: [(recorded_at, value), ...]} for a single device."""
since = datetime.now(timezone.utc) - timedelta(hours=hours)
result = await db.execute(
select(PluginMetric)
.where(
PluginMetric.source_module == "snmp",
PluginMetric.resource_name == device_name,
PluginMetric.recorded_at >= since,
)
.order_by(PluginMetric.recorded_at)
)
rows = result.scalars().all()
out: dict[str, list] = {}
for row in rows:
out.setdefault(row.metric_name, []).append(
(row.recorded_at.isoformat(), row.value)
)
return out
@snmp_bp.get("/")
@require_role(UserRole.viewer)
async def index():
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
async with current_app.db_sessionmaker() as db:
latest = await _latest_readings(db, device_names)
# Merge config + latest readings for the template
devices = []
for cfg in devices_cfg:
name = cfg.get("name") or cfg.get("host", "?")
devices.append({
"name": name,
"host": cfg.get("host", ""),
"oids": cfg.get("oids", []),
"readings": latest.get(name, {}),
})
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
return await render_template("snmp/index.html",
devices=devices,
poll_interval=poll_interval)
def _device_binds_to_host(d: dict, keys: set[str], host_id: str) -> bool:
"""Does config device `d` belong on the host identified by `keys`
(lowercased {address, name}) / `host_id`?
A device may declare an **explicit** binding that decouples "which Steward
host this belongs to" from "where to poll" (the `host` field):
• `host_id` — the Steward Host UUID (exact match), the explicit link.
• `steward_host` — a friendly bind by Host name or address (case-insensitive).
An explicit binding is **exclusive**: a device bound to host A must not also
match host B by a coincidental poll-target string. Only when no explicit
binding is present do we fall back to the implicit `host`-string match
(backward compatible with configs that only set `host`).
"""
explicit_id = str(d.get("host_id", "")).strip()
explicit_name = str(d.get("steward_host", "")).strip().lower()
if explicit_id or explicit_name:
if explicit_id and host_id and explicit_id == host_id:
return True
return bool(explicit_name and explicit_name in keys)
return str(d.get("host", "")).strip().lower() in keys
def _devices_for_host(devices_cfg: list, address: str | None, name: str | None,
host_id: str | None = None) -> list[dict]:
"""SNMP devices that map onto a Steward host's page. Prefers an explicit
per-device binding (`host_id` / `steward_host`); otherwise falls back to
matching the device's `host` (poll target) against the host's address or
name (case-insensitive). See `_device_binds_to_host` for the precedence."""
keys = {(address or "").strip().lower(), (name or "").strip().lower()}
keys.discard("")
hid = (host_id or "").strip()
return [
d for d in devices_cfg
if isinstance(d, dict) and _device_binds_to_host(d, keys, hid)
]
@snmp_bp.get("/host/<host_id>")
@require_role(UserRole.viewer)
async def host_panel(host_id: str):
"""Per-host SNMP fragment for the Hosts hub. Surfaces any configured SNMP
device that maps to this host (by address or name) with its latest readings.
Renders nothing when no device maps, so hosts without SNMP carry no empty card.
"""
from steward.models.hosts import Host
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
async with current_app.db_sessionmaker() as db:
host = (await db.execute(
select(Host).where(Host.id == host_id))).scalar_one_or_none()
if host is None:
return ""
matched = _devices_for_host(devices_cfg, host.address, host.name, host.id)
if not matched:
return ""
names = [d.get("name") or d.get("host", "?") for d in matched]
latest = await _latest_readings(db, names)
devices = [{
"name": d.get("name") or d.get("host", "?"),
"host": d.get("host", ""),
"oids": d.get("oids", []),
"readings": latest.get(d.get("name") or d.get("host", "?"), {}),
"bound": bool(str(d.get("host_id", "")).strip()
or str(d.get("steward_host", "")).strip()),
} for d in matched]
return await render_template("snmp/host_panel.html", devices=devices)
@snmp_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
async with current_app.db_sessionmaker() as db:
latest = await _latest_readings(db, device_names[:10])
devices = []
for cfg in devices_cfg[:10]:
name = cfg.get("name") or cfg.get("host", "?")
devices.append({
"name": name,
"oids": cfg.get("oids", []),
"readings": latest.get(name, {}),
})
return await render_template("snmp/widget.html",
devices=devices,
total=len(devices_cfg))
@snmp_bp.get("/device/<device_name>")
@require_role(UserRole.viewer)
async def device_detail(device_name: str):
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
device_cfg = next(
(d for d in devices_cfg if (d.get("name") or d.get("host")) == device_name),
None,
)
if device_cfg is None:
from quart import abort
abort(404)
hours = int(request.args.get("hours", 24))
async with current_app.db_sessionmaker() as db:
hist = await _history(db, device_name, hours=hours)
import json
history_json = json.dumps(hist)
oid_labels = [o.get("label") or o.get("oid") for o in device_cfg.get("oids", [])]
return await render_template(
"snmp/device.html",
device=device_cfg,
device_name=device_name,
oid_labels=oid_labels,
history_json=history_json,
hours=hours,
)
+69
View File
@@ -0,0 +1,69 @@
# plugins/snmp/scheduler.py
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from quart import Quart
from steward.core.scheduler import ScheduledTask
logger = logging.getLogger(__name__)
def make_poll_task(app: "Quart") -> ScheduledTask:
interval = app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60)
async def poll() -> None:
await _do_poll(app)
return ScheduledTask(
name="snmp_poll",
coro_factory=poll,
interval_seconds=int(interval),
run_on_startup=True,
)
async def _do_poll(app: "Quart") -> None:
from .poller import poll_device
from steward.core.alerts import record_metric
devices: list[dict] = app.config["PLUGINS"]["snmp"].get("devices", [])
if not devices:
return
async with app.db_sessionmaker() as session:
async with session.begin():
for device in devices:
if not isinstance(device, dict):
continue
name = device.get("name") or device.get("host", "unknown")
host = device.get("host", "")
port = int(device.get("port", 161))
community = device.get("community", "public")
version = str(device.get("version", "2c"))
oids = device.get("oids", [])
if not host or not oids:
continue
try:
readings = await poll_device(host, port, community, version, oids)
except Exception:
logger.exception("SNMP poll failed for device %s (%s)", name, host)
continue
for label, value in readings.items():
await record_metric(
session=session,
source_module="snmp",
resource_name=name,
metric_name=label,
value=value,
)
if readings:
logger.debug("SNMP polled %s (%s): %d OID(s)", name, host, len(readings))
else:
logger.debug("SNMP polled %s (%s): no readings", name, host)
+85
View File
@@ -0,0 +1,85 @@
{# plugins/snmp/templates/snmp/device.html #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs %}
{% block title %}SNMP — {{ device_name }} — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("SNMP", "/plugins/snmp/"), (device_name, "")]) }}{% endblock %}
{% block content %}
<div style="display:flex;align-items:center;gap:1rem;margin-bottom:1.5rem;flex-wrap:wrap;">
<h1 class="page-title" style="margin:0;">{{ device_name }}</h1>
<span style="font-size:0.8rem;color:var(--text-dim);">{{ device.host }}</span>
<div style="margin-left:auto;display:flex;gap:0.5rem;">
{% for h in [1, 6, 24, 168] %}
<a href="?hours={{ h }}"
class="btn btn-ghost"
style="font-size:0.78rem;padding:0.2rem 0.6rem;{% if hours == h %}background:var(--accent);color:#fff;border-color:var(--accent);{% endif %}">
{% if h < 24 %}{{ h }}h{% elif h == 24 %}24h{% else %}7d{% endif %}
</a>
{% endfor %}
</div>
</div>
{% if oid_labels %}
<div class="card">
<canvas id="snmp-chart" style="width:100%;max-height:320px;"></canvas>
</div>
<script>
(function(){
const raw = {{ history_json | safe }};
const labels = {{ oid_labels | tojson }};
const palette = ["#6060c0","#e07040","#40b080","#c040c0","#4080e0","#e0c040","#c08040"];
// Collect all unique timestamps, sorted
const tsSet = new Set();
for (const pts of Object.values(raw)) {
for (const [ts] of pts) tsSet.add(ts);
}
const allTs = Array.from(tsSet).sort();
if (allTs.length < 2) {
document.getElementById("snmp-chart").parentElement.innerHTML =
'<p style="color:var(--text-muted);padding:1rem;">Not enough data for a chart yet.</p>';
return;
}
const datasets = [];
labels.forEach((label, i) => {
const pts = raw[label] || [];
const map = Object.fromEntries(pts.map(([ts, v]) => [ts, v]));
datasets.push({
label,
data: allTs.map(ts => map[ts] ?? null),
borderColor: palette[i % palette.length],
backgroundColor: "transparent",
borderWidth: 1.5,
pointRadius: 0,
tension: 0.2,
spanGaps: true,
});
});
const ctx = document.getElementById("snmp-chart").getContext("2d");
new Chart(ctx, {
type: "line",
data: { labels: allTs.map(ts => ts.substring(11, 16)), datasets },
options: {
responsive: true,
interaction: { mode: "index", intersect: false },
plugins: {
legend: { position: "top", labels: { color: "#aaa", boxWidth: 12 } },
tooltip: { backgroundColor: "#1e1e2e", titleColor: "#ccc", bodyColor: "#aaa" },
},
scales: {
x: { ticks: { color: "#888", maxTicksLimit: 12 }, grid: { color: "#2a2a3a" } },
y: { ticks: { color: "#888" }, grid: { color: "#2a2a3a" } },
},
},
});
})();
</script>
{% else %}
<div class="card" style="color:var(--text-muted);text-align:center;padding:2rem;">
No OIDs configured for this device.
</div>
{% endif %}
{% endblock %}
@@ -0,0 +1,51 @@
{# Per-host SNMP fragment — embedded on the Hosts hub via HTMX. Self-contained.
Shows the SNMP device(s) whose address maps to this host + latest readings. #}
<div class="card">
<div style="display:flex;align-items:baseline;justify-content:space-between;margin-bottom:0.6rem;gap:1rem;flex-wrap:wrap;">
<h3 class="section-title" style="margin-bottom:0;">SNMP</h3>
<a href="/plugins/snmp/" style="font-size:0.8rem;color:var(--text-muted);">All devices →</a>
</div>
{% for device in devices %}
<div style="{% if not loop.last %}margin-bottom:1rem;{% endif %}">
<div style="display:flex;align-items:center;gap:0.6rem;margin-bottom:0.5rem;flex-wrap:wrap;">
<span style="font-weight:600;font-size:0.92rem;">{{ device.name }}</span>
<span style="font-size:0.78rem;color:var(--text-dim);font-family:ui-monospace,monospace;">{{ device.host }}</span>
{% if device.bound %}
<span title="Explicitly bound to this host (host_id / steward_host)"
style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:3px;padding:0.05rem 0.4rem;">bound</span>
{% endif %}
{% if device.readings %}
<span style="font-size:0.74rem;color:var(--green);">&#9679; reachable</span>
{% else %}
<span style="font-size:0.74rem;color:var(--text-dim);">&#9675; no data yet</span>
{% endif %}
<a href="/plugins/snmp/device/{{ device.name }}" class="btn btn-ghost btn-sm"
style="margin-left:auto;font-size:0.76rem;padding:0.15rem 0.55rem;">History</a>
</div>
{% if device.readings %}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:0.5rem;">
{% for oid_cfg in device.oids %}
{% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
{% set val = device.readings.get(label) %}
<div style="background:var(--bg-elevated);border-radius:5px;padding:0.5rem 0.7rem;border:1px solid var(--border);">
<div style="font-size:0.68rem;color:var(--text-dim);margin-bottom:0.2rem;font-family:ui-monospace,monospace;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ label }}</div>
{% if val is not none %}
<div style="font-size:1rem;font-weight:600;font-variant-numeric:tabular-nums;">
{% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}<span style="font-size:0.72rem;color:var(--text-muted);">M</span>
{% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}<span style="font-size:0.72rem;color:var(--text-muted);">K</span>
{% else %}{{ "%.4g"|format(val) }}{% endif %}
</div>
{% else %}
<div style="font-size:0.85rem;color:var(--text-dim);"></div>
{% endif %}
</div>
{% endfor %}
</div>
{% else %}
<p style="font-size:0.8rem;color:var(--text-dim);margin:0;">No readings yet — waiting for the next poll.</p>
{% endif %}
</div>
{% endfor %}
</div>
+57
View File
@@ -0,0 +1,57 @@
{# plugins/snmp/templates/snmp/index.html #}
{% extends "base.html" %}
{% block title %}SNMP — Steward{% endblock %}
{% block content %}
<div style="display:flex;align-items:center;gap:1rem;margin-bottom:1.5rem;">
<h1 class="page-title" style="margin:0;">SNMP Devices</h1>
<span style="font-size:0.8rem;color:var(--text-dim);">{{ devices|length }} device(s) configured</span>
</div>
{% if not devices %}
<div class="card" style="color:var(--text-muted);text-align:center;padding:2rem;">
No SNMP devices configured. Add devices to <code>plugin.yaml</code>.
</div>
{% else %}
{% for device in devices %}
<div class="card" style="margin-bottom:1rem;">
<div style="display:flex;align-items:center;gap:1rem;margin-bottom:1rem;">
<h2 style="margin:0;font-size:1rem;font-weight:600;">{{ device.name }}</h2>
<span style="font-size:0.78rem;color:var(--text-dim);">{{ device.host }}</span>
{% if device.readings %}
<span style="font-size:0.75rem;color:var(--green);margin-left:auto;">&#9679; reachable</span>
{% else %}
<span style="font-size:0.75rem;color:var(--text-dim);margin-left:auto;">&#9675; no data yet</span>
{% endif %}
<a href="/plugins/snmp/device/{{ device.name }}" class="btn btn-ghost" style="font-size:0.78rem;padding:0.2rem 0.6rem;">History</a>
</div>
{% if device.readings %}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(200px,1fr));gap:0.75rem;">
{% for oid_cfg in device.oids %}
{% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
{% set val = device.readings.get(label) %}
<div style="background:var(--bg-alt);border-radius:6px;padding:0.6rem 0.9rem;">
<div style="font-size:0.72rem;color:var(--text-dim);margin-bottom:0.2rem;font-family:monospace;">{{ label }}</div>
{% if val is not none %}
<div style="font-size:1.1rem;font-weight:600;font-variant-numeric:tabular-nums;">
{% if val >= 1000000 %}{{ "%.2f"|format(val / 1000000) }}<span style="font-size:0.75rem;color:var(--text-muted);">M</span>
{% elif val >= 1000 %}{{ "%.1f"|format(val / 1000) }}<span style="font-size:0.75rem;color:var(--text-muted);">K</span>
{% else %}{{ "%.4g"|format(val) }}{% endif %}
</div>
{% else %}
<div style="font-size:0.85rem;color:var(--text-dim);"></div>
{% endif %}
</div>
{% endfor %}
</div>
{% else %}
<p style="font-size:0.85rem;color:var(--text-muted);margin:0;">
No readings yet — waiting for next poll cycle.
</p>
{% endif %}
</div>
{% endfor %}
{% endif %}
{% endblock %}
+49
View File
@@ -0,0 +1,49 @@
{# plugins/snmp/templates/snmp/widget.html #}
{% if devices %}
{% for device in devices %}
<div class="ping-row" style="align-items:flex-start;flex-direction:column;gap:0.3rem;padding:0.4rem 0;">
<div style="display:flex;align-items:center;gap:0.6rem;width:100%;">
<span style="font-size:0.82rem;font-weight:500;flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">
{{ device.name }}
</span>
{% if device.readings %}
<span style="font-size:0.7rem;color:var(--green);">&#9679;</span>
{% else %}
<span style="font-size:0.7rem;color:var(--text-dim);">&#9675;</span>
{% endif %}
<a href="/plugins/snmp/device/{{ device.name }}" style="font-size:0.72rem;color:var(--text-muted);">detail →</a>
</div>
{% if device.readings %}
<div style="display:flex;flex-wrap:wrap;gap:0.5rem 1rem;padding-left:0.1rem;">
{% for oid_cfg in device.oids[:4] %}
{% set label = oid_cfg.label if oid_cfg.label else oid_cfg.oid %}
{% set val = device.readings.get(label) %}
{% if val is not none %}
<span style="font-size:0.78rem;font-variant-numeric:tabular-nums;">
<span style="color:var(--text-muted);">{{ label }}</span>
<span style="margin-left:0.25rem;">
{% if val >= 1000000 %}{{ "%.1f"|format(val / 1000000) }}M
{% elif val >= 1000 %}{{ "%.0f"|format(val / 1000) }}K
{% else %}{{ "%.4g"|format(val) }}{% endif %}
</span>
</span>
{% endif %}
{% endfor %}
{% if device.oids|length > 4 %}
<span style="font-size:0.72rem;color:var(--text-dim);">+{{ device.oids|length - 4 }} more</span>
{% endif %}
</div>
{% endif %}
</div>
{% if not loop.last %}
<div style="height:1px;background:var(--border-low);margin:0.1rem 0;"></div>
{% endif %}
{% endfor %}
{% if total > devices|length %}
<div style="color:var(--text-dim);font-size:0.75rem;padding:0.25rem 0;">
+{{ total - devices|length }} more — <a href="/plugins/snmp/">view all →</a>
</div>
{% endif %}
{% else %}
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No SNMP devices configured.</p>
{% endif %}
+35
View File
@@ -0,0 +1,35 @@
# plugins/traefik/__init__.py
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from quart import Quart
_app: "Quart | None" = None
def setup(app: "Quart") -> None:
global _app
_app = app
from .models import TraefikMetric, TraefikCert, TraefikGlobalStat, TraefikAccessSummary # noqa: F401
def get_scheduled_tasks() -> list:
from .scheduler import make_scrape_task, make_access_log_task
tasks = [make_scrape_task(_app)]
access_log_cfg = _app.config["PLUGINS"]["traefik"].get("access_log", {})
if not isinstance(access_log_cfg, dict):
access_log_cfg = {}
if access_log_cfg.get("enabled", False):
tasks.append(make_access_log_task(_app))
return tasks
def get_blueprint():
from .routes import traefik_bp
return traefik_bp
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (Traefik services view).
return [{"label": "Traefik", "href": "/plugins/traefik/"}]
+199
View File
@@ -0,0 +1,199 @@
# plugins/traefik/access_log.py
"""Traefik JSON access log tailer and traffic-origin aggregator.
Reads new lines from Traefik's JSON access log on each scheduler tick,
classifies source IPs as internal (RFC1918) or external, optionally
looks up country codes via MaxMind GeoLite2, and returns an aggregated
summary ready to persist in TraefikAccessSummary.
"""
from __future__ import annotations
import ipaddress
import json
from collections import defaultdict
from pathlib import Path
# ── Module-level file-position state ─────────────────────────────────────────
_log_position: int = 0 # byte offset of last read
_log_inode: int = 0 # detect rotation by inode change
# ── Private IP ranges (RFC1918 + loopback + link-local) ──────────────────────
_PRIVATE_NETS = [
ipaddress.ip_network("10.0.0.0/8"),
ipaddress.ip_network("172.16.0.0/12"),
ipaddress.ip_network("192.168.0.0/16"),
ipaddress.ip_network("127.0.0.0/8"),
ipaddress.ip_network("169.254.0.0/16"),
ipaddress.ip_network("::1/128"),
ipaddress.ip_network("fc00::/7"),
ipaddress.ip_network("fe80::/10"),
]
def _strip_port(raw: str) -> str:
"""Strip trailing :port from an IPv4 address string. Leaves IPv6 untouched."""
if not raw:
return raw
# Bracketed IPv6 with port: [::1]:12345
if raw.startswith("["):
return raw[1:raw.index("]")] if "]" in raw else raw
# IPv4:port has exactly one colon
if raw.count(":") == 1:
return raw.rsplit(":", 1)[0]
return raw
def is_internal(raw_ip: str) -> bool:
"""Return True if the address falls within a private/loopback range."""
ip_str = _strip_port(raw_ip)
try:
addr = ipaddress.ip_address(ip_str)
return any(addr in net for net in _PRIVATE_NETS)
except ValueError:
return False
def _country(ip_str: str, geoip_db) -> str | None:
"""Return ISO country code via an open maxminddb reader, or None."""
if geoip_db is None:
return None
clean = _strip_port(ip_str)
try:
record = geoip_db.get(clean)
if record:
return record.get("country", {}).get("iso_code")
except Exception:
pass
return None
def read_new_lines(log_path: str) -> list[dict]:
"""Return all new JSON-parsed access log entries since the last call.
Handles log rotation: if the file shrinks or its inode changes, we
reset to the beginning of the new file.
"""
global _log_position, _log_inode
path = Path(log_path)
if not path.exists():
return []
try:
stat = path.stat()
except OSError:
return []
current_inode = stat.st_ino
current_size = stat.st_size
# Rotation detected
if current_inode != _log_inode or current_size < _log_position:
_log_position = 0
_log_inode = current_inode
if current_size == _log_position:
return []
entries: list[dict] = []
try:
with open(log_path, "rb") as f:
f.seek(_log_position)
raw = f.read(current_size - _log_position)
_log_position = current_size
except OSError:
return []
for raw_line in raw.split(b"\n"):
raw_line = raw_line.strip()
if not raw_line:
continue
try:
entries.append(json.loads(raw_line))
except json.JSONDecodeError:
continue
return entries
def aggregate(entries: list[dict], geoip_db=None, top_n: int = 10) -> dict | None:
"""Aggregate a batch of access log entries into a summary dict.
Returns None if there are no entries.
Result keys map directly to TraefikAccessSummary columns (excluding
id and period_start/period_end, which the caller provides).
"""
if not entries:
return None
total = 0
internal = 0
external = 0
total_bytes = 0.0
by_router: dict[str, int] = defaultdict(int)
ip_counts: dict[str, int] = defaultdict(int)
ip_internal: dict[str, bool] = {}
ip_country: dict[str, str | None] = {}
country_counts: dict[str, int] = defaultdict(int)
for entry in entries:
total += 1
raw_ip = entry.get("ClientHost", "")
router = entry.get("RouterName") or entry.get("ServiceName") or "unknown"
total_bytes += entry.get("DownstreamContentSize") or 0
by_router[router] += 1
internal_flag = is_internal(raw_ip)
if internal_flag:
internal += 1
else:
external += 1
if raw_ip not in ip_country:
ip_country[raw_ip] = _country(raw_ip, geoip_db)
c = ip_country[raw_ip]
if c:
country_counts[c] += 1
ip_counts[raw_ip] += 1
ip_internal[raw_ip] = internal_flag
top_ips = [
{
"ip": ip,
"count": cnt,
"internal": ip_internal[ip],
"country": ip_country.get(ip),
}
for ip, cnt in sorted(ip_counts.items(), key=lambda x: x[1], reverse=True)[:top_n]
]
top_countries = [
{"country": c, "count": n}
for c, n in sorted(country_counts.items(), key=lambda x: x[1], reverse=True)[:top_n]
]
top_routers = [
{"router": r, "count": n}
for r, n in sorted(by_router.items(), key=lambda x: x[1], reverse=True)[:top_n]
]
return {
"total_requests": total,
"internal_requests": internal,
"external_requests": external,
"total_bytes": total_bytes,
"top_ips_json": json.dumps(top_ips),
"top_countries_json": json.dumps(top_countries),
"top_routers_json": json.dumps(top_routers),
}
def open_geoip(db_path: str | None):
"""Open a MaxMind mmdb file if available. Returns None if not configured or missing."""
if not db_path:
return None
try:
import maxminddb # type: ignore[import]
return maxminddb.open_database(db_path)
except Exception:
return None
+75
View File
@@ -0,0 +1,75 @@
# plugins/traefik/migrations/env.py
"""Alembic env.py for the Traefik plugin.
For standalone use during plugin development.
At app startup, migration_runner.py uses the core env.py with version_locations.
"""
import asyncio
import os
import sys
from pathlib import Path
from logging.config import fileConfig
from sqlalchemy import pool
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import async_engine_from_config
from alembic import context
# Make steward importable
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
from steward.models.base import Base
import steward.models # noqa: F401 — core models
from plugins.traefik.models import TraefikMetric # noqa: F401 — plugin model
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def _get_url() -> str:
import yaml
cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml")
try:
with open(cfg_path) as f:
cfg = yaml.safe_load(f) or {}
url = cfg.get("database", {}).get("url", "")
except FileNotFoundError:
url = ""
return os.environ.get("STEWARD_DATABASE__URL", url)
def run_migrations_offline() -> None:
context.configure(
url=_get_url(), target_metadata=target_metadata,
literal_binds=True, dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def do_run_migrations(connection: Connection) -> None:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
cfg = config.get_section(config.config_ini_section, {})
cfg["sqlalchemy.url"] = _get_url()
connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
asyncio.run(run_async_migrations())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
@@ -0,0 +1,37 @@
# plugins/traefik/migrations/versions/traefik_001_initial.py
"""Traefik metrics table
Revision ID: traefik_001_initial
Revises: (none — this is a branch off the core head via depends_on)
Create Date: 2026-03-17
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "traefik_001_initial"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = "traefik"
depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
def upgrade() -> None:
op.create_table(
"traefik_metrics",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("router_name", sa.String(255), nullable=False),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("request_rate", sa.Float, nullable=False),
sa.Column("error_rate_4xx_pct", sa.Float, nullable=False),
sa.Column("error_rate_5xx_pct", sa.Float, nullable=False),
sa.Column("latency_p50_ms", sa.Float, nullable=False),
sa.Column("latency_p95_ms", sa.Float, nullable=False),
sa.Column("latency_p99_ms", sa.Float, nullable=False),
)
op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics",
["router_name", "scraped_at"])
def downgrade() -> None:
op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics")
op.drop_table("traefik_metrics")
@@ -0,0 +1,48 @@
"""Traefik extended stats: bandwidth, certs, global stats
Revision ID: traefik_002_stats
Revises: traefik_001_initial
Create Date: 2026-03-19
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "traefik_002_stats"
down_revision: Union[str, None] = "traefik_001_initial"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"traefik_metrics",
sa.Column("response_bytes_rate", sa.Float, nullable=False, server_default="0"),
)
op.create_table(
"traefik_certs",
sa.Column("serial", sa.String(255), primary_key=True),
sa.Column("cn", sa.String(512), nullable=True),
sa.Column("sans", sa.Text, nullable=True),
sa.Column("not_after", sa.DateTime(timezone=True), nullable=False),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_table(
"traefik_global_stats",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("open_conns_total", sa.Float, nullable=True),
sa.Column("config_reloads_total", sa.Float, nullable=True),
sa.Column("config_last_reload_success", sa.Float, nullable=True),
sa.Column("process_memory_bytes", sa.Float, nullable=True),
)
op.create_index("ix_traefik_global_stats_scraped", "traefik_global_stats", ["scraped_at"])
def downgrade() -> None:
op.drop_index("ix_traefik_global_stats_scraped", "traefik_global_stats")
op.drop_table("traefik_global_stats")
op.drop_table("traefik_certs")
op.drop_column("traefik_metrics", "response_bytes_rate")
@@ -0,0 +1,40 @@
"""Traefik access log traffic summaries
Revision ID: traefik_003_access_log
Revises: traefik_002_stats
Create Date: 2026-03-20
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "traefik_003_access_log"
down_revision: Union[str, None] = "traefik_002_stats"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"traefik_access_summaries",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("period_start", sa.DateTime(timezone=True), nullable=False),
sa.Column("period_end", sa.DateTime(timezone=True), nullable=False),
sa.Column("total_requests", sa.Integer, nullable=False, server_default="0"),
sa.Column("internal_requests", sa.Integer, nullable=False, server_default="0"),
sa.Column("external_requests", sa.Integer, nullable=False, server_default="0"),
sa.Column("total_bytes", sa.Float, nullable=False, server_default="0"),
sa.Column("top_ips_json", sa.Text, nullable=False, server_default="[]"),
sa.Column("top_countries_json", sa.Text, nullable=False, server_default="[]"),
sa.Column("top_routers_json", sa.Text, nullable=False, server_default="[]"),
)
op.create_index(
"ix_traefik_access_summaries_period_end",
"traefik_access_summaries",
["period_end"],
)
def downgrade() -> None:
op.drop_index("ix_traefik_access_summaries_period_end", "traefik_access_summaries")
op.drop_table("traefik_access_summaries")
+67
View File
@@ -0,0 +1,67 @@
# plugins/traefik/models.py
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from sqlalchemy import DateTime, Float, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from steward.models.base import Base
class TraefikMetric(Base):
__tablename__ = "traefik_metrics"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
router_name: Mapped[str] = mapped_column(String(255), nullable=False)
scraped_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
)
request_rate: Mapped[float] = mapped_column(Float, nullable=False)
error_rate_4xx_pct: Mapped[float] = mapped_column(Float, nullable=False)
error_rate_5xx_pct: Mapped[float] = mapped_column(Float, nullable=False)
latency_p50_ms: Mapped[float] = mapped_column(Float, nullable=False)
latency_p95_ms: Mapped[float] = mapped_column(Float, nullable=False)
latency_p99_ms: Mapped[float] = mapped_column(Float, nullable=False)
response_bytes_rate: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
class TraefikCert(Base):
"""TLS certificate expiry tracking — upserted by serial each scrape."""
__tablename__ = "traefik_certs"
serial: Mapped[str] = mapped_column(String(255), primary_key=True)
cn: Mapped[str | None] = mapped_column(String(512), nullable=True)
sans: Mapped[str | None] = mapped_column(Text, nullable=True)
not_after: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
scraped_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
)
class TraefikAccessSummary(Base):
"""Aggregated traffic-origin summary — one row per scheduler tick."""
__tablename__ = "traefik_access_summaries"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
period_start: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
period_end: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
total_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
internal_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
external_requests: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
total_bytes: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
top_ips_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
top_countries_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
top_routers_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
class TraefikGlobalStat(Base):
"""One row per scrape — process-level and config stats."""
__tablename__ = "traefik_global_stats"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
scraped_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
)
open_conns_total: Mapped[float | None] = mapped_column(Float, nullable=True)
config_reloads_total: Mapped[float | None] = mapped_column(Float, nullable=True)
config_last_reload_success: Mapped[float | None] = mapped_column(Float, nullable=True)
process_memory_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)
+21
View File
@@ -0,0 +1,21 @@
# plugins/traefik/plugin.yaml
name: traefik
version: "1.0.0"
description: "Traefik reverse proxy metrics and access log integration"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/traefik"
tags:
- proxy
- metrics
- access-log
config:
metrics_url: "http://localhost:8080/metrics"
scrape_interval_seconds: 60
access_log:
enabled: false
path: "/var/log/traefik/access.log"
geoip_db: "" # optional: path to MaxMind GeoLite2-Country.mmdb
+373
View File
@@ -0,0 +1,373 @@
# plugins/traefik/routes.py
from __future__ import annotations
from collections import defaultdict
from datetime import datetime, timezone
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import Integer, cast, func, select
import json
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric
traefik_bp = Blueprint("traefik", __name__, template_folder="templates")
def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
"""Generate an inline SVG sparkline from a list of float values."""
if len(values) < 2:
return f'<svg width="{width}" height="{height}"></svg>'
mn, mx = min(values), max(values)
if mx == mn:
mx = mn + 1.0
step = width / (len(values) - 1)
pts = []
for i, v in enumerate(values):
x = i * step
y = height - (v - mn) / (mx - mn) * (height - 2) - 1
pts.append(f"{x:.1f},{y:.1f}")
poly = " ".join(pts)
return (
f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
f'style="vertical-align:middle;">'
f'<polyline points="{poly}" fill="none" stroke="#6060c0" stroke-width="1.5"/>'
f'</svg>'
)
@traefik_bp.get("/")
@require_role(UserRole.viewer)
async def index():
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
_al_cfg = current_app.config["PLUGINS"]["traefik"].get("access_log", {})
if not isinstance(_al_cfg, dict):
_al_cfg = {}
access_log_enabled = _al_cfg.get("enabled", False)
current_range = request.args.get("range", DEFAULT_RANGE)
return await render_template(
"traefik/index.html",
poll_interval=poll_interval,
access_log_enabled=access_log_enabled,
current_range=current_range,
)
@traefik_bp.get("/rows")
@require_role(UserRole.viewer)
async def rows():
"""HTMX fragment: service cards with sparklines scoped to selected time range."""
since, range_key = parse_range(request.args.get("range"))
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
# All known routers
result = await db.execute(
select(TraefikMetric.router_name)
.distinct()
.order_by(TraefikMetric.router_name)
)
routers = [row[0] for row in result.all()]
b_secs = bucket_seconds(since)
bucket_col = (
cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs
).label("bucket")
router_data = []
for router in routers:
# Bucketed history for sparklines — always ~80 points regardless of range
result = await db.execute(
select(
func.avg(TraefikMetric.request_rate).label("request_rate"),
func.avg(TraefikMetric.latency_p95_ms).label("latency_p95_ms"),
func.avg(TraefikMetric.error_rate_5xx_pct).label("error_rate_5xx_pct"),
func.min(TraefikMetric.scraped_at).label("scraped_at"),
bucket_col,
)
.where(TraefikMetric.router_name == router)
.where(TraefikMetric.scraped_at >= since)
.group_by(bucket_col)
.order_by(bucket_col)
)
history = result.all()
# Latest value — always the most recent raw row
result = await db.execute(
select(TraefikMetric)
.where(TraefikMetric.router_name == router)
.order_by(TraefikMetric.scraped_at.desc())
.limit(1)
)
latest = result.scalar_one_or_none()
if not latest:
continue
router_data.append({
"name": router,
"latest": latest,
"sparkline_req": _sparkline([r.request_rate for r in history]),
"sparkline_p95": _sparkline([r.latency_p95_ms for r in history]),
"sparkline_5xx": _sparkline([r.error_rate_5xx_pct for r in history]),
})
# Latest global stat (always most recent, not range-filtered)
result = await db.execute(
select(TraefikGlobalStat)
.order_by(TraefikGlobalStat.scraped_at.desc())
.limit(1)
)
global_stat = result.scalar_one_or_none()
# All certs, sorted by soonest expiry
result = await db.execute(
select(TraefikCert).order_by(TraefikCert.not_after)
)
raw_certs = result.scalars().all()
certs = []
for c in raw_certs:
days = (c.not_after - now).total_seconds() / 86400.0
certs.append({
"serial": c.serial,
"cn": c.cn or c.serial,
"sans": c.sans,
"not_after": c.not_after,
"days_remaining": days,
})
return await render_template(
"traefik/rows.html",
router_data=router_data,
global_stat=global_stat,
certs=certs,
range_key=range_key,
)
@traefik_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""HTMX dashboard widget fragment."""
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(TraefikMetric.router_name).distinct().order_by(TraefikMetric.router_name)
)
routers = [row[0] for row in result.all()]
router_data = []
for router in routers:
result = await db.execute(
select(TraefikMetric)
.where(TraefikMetric.router_name == router)
.order_by(TraefikMetric.scraped_at.desc())
.limit(1)
)
latest = result.scalar_one_or_none()
if latest:
router_data.append({"name": router, "latest": latest})
router_data.sort(key=lambda r: (
-(1 if r["latest"].error_rate_5xx_pct > 0.1 else 0),
-(1 if r["latest"].error_rate_4xx_pct > 5.0 else 0),
-r["latest"].request_rate,
))
total_routers = len(router_data)
router_data = router_data[:5]
result = await db.execute(
select(TraefikCert).order_by(TraefikCert.not_after)
)
raw_certs = result.scalars().all()
expiring_certs = []
for c in raw_certs:
days = (c.not_after - now).total_seconds() / 86400.0
if days < 30:
expiring_certs.append({"cn": c.cn or c.serial, "days_remaining": days})
return await render_template(
"traefik/widget.html",
router_data=router_data,
expiring_certs=expiring_certs,
total_routers=total_routers,
)
@traefik_bp.get("/widget/access_log")
@require_role(UserRole.viewer)
async def widget_access_log():
"""HTMX dashboard widget: traffic origin summary with IP filter."""
from collections import defaultdict
ip_filter = request.args.get("ip_filter", "all")
limit = max(1, min(50, int(request.args.get("limit", 10) or 10)))
widget_id = request.args.get("wid", "0")
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(TraefikAccessSummary)
.order_by(TraefikAccessSummary.period_end.desc())
.limit(12) # last ~1 hour of 5-min windows
)
rows = result.scalars().all()
if not rows:
return await render_template("traefik/widget_access_log.html",
summary=None, widget_id=widget_id)
total = sum(r.total_requests for r in rows)
internal = sum(r.internal_requests for r in rows)
external = sum(r.external_requests for r in rows)
ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True})
for row in rows:
for entry in json.loads(row.top_ips_json):
k = entry["ip"]
ip_counts[k]["count"] += entry["count"]
ip_counts[k]["internal"] = entry.get("internal", True)
all_ips = sorted(
[{"ip": k, **v} for k, v in ip_counts.items()],
key=lambda x: x["count"], reverse=True,
)
if ip_filter == "internal":
all_ips = [e for e in all_ips if e["internal"]]
elif ip_filter == "external":
all_ips = [e for e in all_ips if not e["internal"]]
summary = {
"total": total,
"internal": internal,
"external": external,
"external_pct": (external / total * 100.0) if total else 0.0,
"top_ips": all_ips[:limit],
"ip_filter": ip_filter,
}
return await render_template("traefik/widget_access_log.html",
summary=summary, widget_id=widget_id)
@traefik_bp.get("/widget/request_chart")
@require_role(UserRole.viewer)
async def widget_request_chart():
"""HTMX dashboard widget: Chart.js request rate + error % over time."""
from datetime import timedelta
hours = max(1, min(24, int(request.args.get("hours", 6) or 6)))
widget_id = request.args.get("wid", "0")
since = datetime.now(timezone.utc) - timedelta(hours=hours)
b_secs = max(60, (hours * 3600) // 80) # ~80 buckets
bucket_col = (
cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs
).label("bucket")
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(
func.avg(TraefikMetric.request_rate).label("req_rate"),
func.avg(TraefikMetric.error_rate_5xx_pct).label("err_pct"),
func.min(TraefikMetric.scraped_at).label("scraped_at"),
bucket_col,
)
.where(TraefikMetric.scraped_at >= since)
.group_by(bucket_col)
.order_by(bucket_col)
)
history = result.all()
labels = list(range(len(history)))
req_rates = [round(r.req_rate or 0, 3) for r in history]
err_pcts = [round(r.err_pct or 0, 2) for r in history]
return await render_template(
"traefik/widget_request_chart.html",
labels_json=json.dumps(labels),
req_rate_json=json.dumps(req_rates),
error_rate_json=json.dumps(err_pcts),
widget_id=widget_id,
hours=hours,
)
@traefik_bp.get("/traffic")
@require_role(UserRole.viewer)
async def traffic():
"""HTMX fragment: traffic origin summary scoped to selected time range."""
since, range_key = parse_range(request.args.get("range"))
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(TraefikAccessSummary)
.where(TraefikAccessSummary.period_end >= since)
.order_by(TraefikAccessSummary.period_end.asc())
# No LIMIT — all summaries in range needed for accurate totals;
# sparkline subsamples to 80 points afterwards
)
rows = result.scalars().all()
if not rows:
return await render_template(
"traefik/traffic.html", summary=None, history=[], range_key=range_key
)
total = sum(r.total_requests for r in rows)
internal = sum(r.internal_requests for r in rows)
external = sum(r.external_requests for r in rows)
total_bytes = sum(r.total_bytes for r in rows)
ip_counts: dict = defaultdict(lambda: {"count": 0, "internal": True, "country": None})
country_counts: dict[str, int] = defaultdict(int)
router_counts: dict[str, int] = defaultdict(int)
for row in rows:
for entry in json.loads(row.top_ips_json):
k = entry["ip"]
ip_counts[k]["count"] += entry["count"]
ip_counts[k]["internal"] = entry["internal"]
ip_counts[k]["country"] = entry.get("country")
for entry in json.loads(row.top_countries_json):
country_counts[entry["country"]] += entry["count"]
for entry in json.loads(row.top_routers_json):
router_counts[entry["router"]] += entry["count"]
top_ips = sorted(
[{"ip": k, **v} for k, v in ip_counts.items()],
key=lambda x: x["count"], reverse=True
)[:10]
top_countries = sorted(
[{"country": k, "count": v} for k, v in country_counts.items()],
key=lambda x: x["count"], reverse=True
)[:10]
top_routers = sorted(
[{"router": k, "count": v} for k, v in router_counts.items()],
key=lambda x: x["count"], reverse=True
)[:10]
history = [
{
"period_end": r.period_end,
"external_pct": (r.external_requests / r.total_requests * 100.0)
if r.total_requests else 0.0,
}
for r in rows
]
summary = {
"total": total,
"internal": internal,
"external": external,
"total_bytes": total_bytes,
"external_pct": (external / total * 100.0) if total else 0.0,
"top_ips": top_ips,
"top_countries": top_countries,
"top_routers": top_routers,
"sparkline_external": _sparkline(
[h["external_pct"] for h in subsample(history, 80)]
),
}
return await render_template(
"traefik/traffic.html", summary=summary, history=history, range_key=range_key
)
+203
View File
@@ -0,0 +1,203 @@
# plugins/traefik/scheduler.py
from __future__ import annotations
import logging
import time
from typing import TYPE_CHECKING
from datetime import datetime
from steward.core.scheduler import ScheduledTask
if TYPE_CHECKING:
from quart import Quart
logger = logging.getLogger(__name__)
# Module-level state: previous scrape data for delta computation
_prev_metrics: dict = {}
_prev_time: float = 0.0
# Access log state
_geoip_db = None
_access_log_last_tick: "datetime | None" = None
def make_scrape_task(app: "Quart") -> ScheduledTask:
"""Return a ScheduledTask that scrapes Traefik metrics on the configured interval."""
interval = app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
async def scrape() -> None:
await _do_scrape(app)
return ScheduledTask(
name="traefik_scrape",
coro_factory=scrape,
interval_seconds=int(interval),
run_on_startup=True,
)
async def _do_scrape(app: "Quart") -> None:
"""Fetch Traefik metrics, write to DB, and emit plugin_metrics."""
global _prev_metrics, _prev_time
from datetime import datetime, timezone
from .models import TraefikCert, TraefikGlobalStat, TraefikMetric
from .scraper import (
compute_global_stats,
compute_router_metrics,
extract_certs,
fetch_metrics,
)
from steward.core.alerts import record_metric
metrics_url: str = app.config["PLUGINS"]["traefik"]["metrics_url"]
try:
current = await fetch_metrics(metrics_url)
except Exception:
logger.exception("Traefik scrape failed (url=%s)", metrics_url)
return
now = time.monotonic()
elapsed = now - _prev_time if _prev_time else 60.0
router_metrics = compute_router_metrics(current, _prev_metrics or None, elapsed)
global_stats = compute_global_stats(current, _prev_metrics or None)
cert_list = extract_certs(current)
_prev_metrics = current
_prev_time = now
scraped_at = datetime.now(timezone.utc)
async with app.db_sessionmaker() as session:
async with session.begin():
# ── Per-service metrics ───────────────────────────────────────────
for router_name, metrics in router_metrics.items():
row = TraefikMetric(
router_name=router_name,
scraped_at=scraped_at,
request_rate=metrics["request_rate"],
error_rate_4xx_pct=metrics["error_rate_4xx_pct"],
error_rate_5xx_pct=metrics["error_rate_5xx_pct"],
latency_p50_ms=metrics["latency_p50_ms"],
latency_p95_ms=metrics["latency_p95_ms"],
latency_p99_ms=metrics["latency_p99_ms"],
response_bytes_rate=metrics["response_bytes_rate"],
)
session.add(row)
for metric_name, value in [
("request_rate", metrics["request_rate"]),
("error_rate_4xx_pct", metrics["error_rate_4xx_pct"]),
("error_rate_5xx_pct", metrics["error_rate_5xx_pct"]),
("latency_p50_ms", metrics["latency_p50_ms"]),
("latency_p95_ms", metrics["latency_p95_ms"]),
("latency_p99_ms", metrics["latency_p99_ms"]),
("response_bytes_rate", metrics["response_bytes_rate"]),
]:
await record_metric(
session=session,
source_module="traefik",
resource_name=router_name,
metric_name=metric_name,
value=value,
)
# ── Global stats ──────────────────────────────────────────────────
session.add(TraefikGlobalStat(scraped_at=scraped_at, **global_stats))
# ── TLS certs (upsert by serial) ──────────────────────────────────
for cert in cert_list:
existing = await session.get(TraefikCert, cert["serial"])
if existing:
existing.cn = cert["cn"]
existing.sans = cert["sans"]
existing.not_after = cert["not_after"]
existing.scraped_at = scraped_at
else:
session.add(TraefikCert(
serial=cert["serial"],
cn=cert["cn"],
sans=cert["sans"],
not_after=cert["not_after"],
scraped_at=scraped_at,
))
# Emit cert_expiry_days for alert rules
days_remaining = (cert["not_after"] - scraped_at).total_seconds() / 86400.0
await record_metric(
session=session,
source_module="traefik",
resource_name=cert["cn"] or cert["serial"],
metric_name="cert_expiry_days",
value=max(0.0, days_remaining),
)
logger.debug(
"Traefik scrape: %d router(s), %d cert(s)",
len(router_metrics),
len(cert_list),
)
def make_access_log_task(app: "Quart") -> ScheduledTask:
"""Return a ScheduledTask that processes new Traefik access log lines."""
global _geoip_db
cfg = app.config["PLUGINS"]["traefik"].get("access_log", {})
if not isinstance(cfg, dict):
cfg = {}
log_path: str = cfg.get("path", "/var/log/traefik/access.log")
geoip_path: str = cfg.get("geoip_db", "")
from .access_log import open_geoip
_geoip_db = open_geoip(geoip_path or None)
if _geoip_db:
logger.info("Traefik access log: GeoIP database loaded from %s", geoip_path)
else:
logger.info("Traefik access log: GeoIP not configured, country lookup disabled")
interval = app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]
async def process() -> None:
await _do_access_log(app, log_path)
return ScheduledTask(
name="traefik_access_log",
coro_factory=process,
interval_seconds=int(interval),
run_on_startup=True,
)
async def _do_access_log(app: "Quart", log_path: str) -> None:
"""Read new access log lines, aggregate, and persist a summary."""
global _access_log_last_tick
from datetime import timezone
from .access_log import read_new_lines, aggregate
from .models import TraefikAccessSummary
now = datetime.now(timezone.utc)
period_start = _access_log_last_tick or now
_access_log_last_tick = now
entries = read_new_lines(log_path)
summary = aggregate(entries, geoip_db=_geoip_db)
if summary is None:
return
async with app.db_sessionmaker() as session:
async with session.begin():
session.add(TraefikAccessSummary(
period_start=period_start,
period_end=now,
**summary,
))
logger.debug(
"Traefik access log: %d requests (%d internal, %d external)",
summary["total_requests"],
summary["internal_requests"],
summary["external_requests"],
)
+276
View File
@@ -0,0 +1,276 @@
# plugins/traefik/scraper.py
"""Prometheus text-format scraper and per-service metrics calculator for Traefik.
Traefik exposes Prometheus metrics at /metrics. This module:
1. Fetches the endpoint via httpx
2. Parses the Prometheus text format (counters + histograms)
3. Computes per-service request rate (delta/elapsed), error rates (% 4xx, % 5xx),
latency percentiles (p50, p95, p99) via linear histogram interpolation,
and bandwidth (bytes/sec from response bytes counter delta)
4. Extracts TLS certificate expiry info
5. Computes process-level and config health stats
"""
from __future__ import annotations
import math
import re
from collections import defaultdict
from datetime import datetime, timezone
import httpx
# ──────────────────────────────────────────────────────────────────────────────
# Prometheus text format parser
# ──────────────────────────────────────────────────────────────────────────────
# Parsed sample: {metric_name: {frozenset(label_items): float}}
ParsedMetrics = dict[str, dict[frozenset, float]]
_SAMPLE_RE = re.compile(
r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+'
r'([+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?|[+-]?Inf|NaN)'
)
_LABEL_RE = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"\\]*(?:\\.[^"\\]*)*)"')
def parse_prometheus(text: str) -> ParsedMetrics:
"""Parse Prometheus text exposition format into nested dicts.
Returns: {metric_name: {frozenset({(label, value), ...}): float}}
"""
metrics: ParsedMetrics = {}
for line in text.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
m = _SAMPLE_RE.match(line)
if not m:
continue
name, labels_str, value_str = m.groups()
try:
value = float(value_str)
except ValueError:
continue
labels = frozenset(
(lm.group(1), lm.group(2))
for lm in _LABEL_RE.finditer(labels_str or "")
)
metrics.setdefault(name, {})[labels] = value
return metrics
# ──────────────────────────────────────────────────────────────────────────────
# Per-router metric computation
# ──────────────────────────────────────────────────────────────────────────────
def compute_router_metrics(
current: ParsedMetrics,
previous: ParsedMetrics | None,
elapsed_seconds: float,
) -> dict[str, dict[str, float]]:
"""Compute per-service derived metrics from two consecutive scrapes.
Uses traefik_service_* metrics (available by default).
Returns:
{service_name: {
"request_rate": float, # requests/sec
"error_rate_4xx_pct": float, # % 4xx of total
"error_rate_5xx_pct": float, # % 5xx of total
"latency_p50_ms": float,
"latency_p95_ms": float,
"latency_p99_ms": float,
"response_bytes_rate": float, # bytes/sec
}}
"""
prev = previous or {}
elapsed = max(elapsed_seconds, 1.0)
# ── Request counters ──────────────────────────────────────────────────────
req_samples = current.get("traefik_service_requests_total", {})
prev_req = prev.get("traefik_service_requests_total", {})
service_total: dict[str, float] = defaultdict(float)
service_4xx: dict[str, float] = defaultdict(float)
service_5xx: dict[str, float] = defaultdict(float)
for labels, value in req_samples.items():
label_dict = dict(labels)
service = label_dict.get("service", "")
if not service:
continue
code = label_dict.get("code", "")
prev_value = prev_req.get(labels, value) # no delta on first scrape
delta = max(0.0, value - prev_value)
service_total[service] += delta
if code.startswith("4"):
service_4xx[service] += delta
elif code.startswith("5"):
service_5xx[service] += delta
# ── Response bytes counters ───────────────────────────────────────────────
bytes_samples = current.get("traefik_service_responses_bytes_total", {})
prev_bytes = prev.get("traefik_service_responses_bytes_total", {})
service_bytes: dict[str, float] = defaultdict(float)
for labels, value in bytes_samples.items():
service = dict(labels).get("service", "")
if not service:
continue
prev_value = prev_bytes.get(labels, value)
service_bytes[service] += max(0.0, value - prev_value)
# ── Histogram buckets ─────────────────────────────────────────────────────
hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {})
# Collect all known services
all_services: set[str] = set(service_total.keys()) | set(service_bytes.keys())
for labels in hist_buckets:
service = dict(labels).get("service", "")
if service:
all_services.add(service)
result: dict[str, dict[str, float]] = {}
for service in all_services:
total = service_total.get(service, 0.0)
result[service] = {
"request_rate": total / elapsed,
"error_rate_4xx_pct": (service_4xx.get(service, 0.0) / total * 100.0)
if total > 0 else 0.0,
"error_rate_5xx_pct": (service_5xx.get(service, 0.0) / total * 100.0)
if total > 0 else 0.0,
"latency_p50_ms": _percentile_ms(hist_buckets, service, 0.50),
"latency_p95_ms": _percentile_ms(hist_buckets, service, 0.95),
"latency_p99_ms": _percentile_ms(hist_buckets, service, 0.99),
"response_bytes_rate": service_bytes.get(service, 0.0) / elapsed,
}
return result
def compute_global_stats(
current: ParsedMetrics,
previous: ParsedMetrics | None,
) -> dict[str, float | None]:
"""Extract process-level and config health metrics from a scrape.
Returns:
{
"open_conns_total": float | None,
"config_reloads_total": float | None,
"config_last_reload_success": float | None, # 1.0 = success, 0.0 = failure
"process_memory_bytes": float | None,
}
"""
stats: dict[str, float | None] = {
"open_conns_total": None,
"config_reloads_total": None,
"config_last_reload_success": None,
"process_memory_bytes": None,
}
# Open connections — sum across all entrypoints/protocols
open_conns = current.get("traefik_open_connections", {})
if open_conns:
stats["open_conns_total"] = sum(open_conns.values())
# Config reloads — sum across result labels (success + failure)
reloads = current.get("traefik_config_reloads_total", {})
if reloads:
stats["config_reloads_total"] = sum(reloads.values())
# Last reload success gauge
last_success = current.get("traefik_config_last_reload_success", {})
if last_success:
# Usually a single sample with no labels
stats["config_last_reload_success"] = next(iter(last_success.values()))
# Process RSS memory
mem = current.get("process_resident_memory_bytes", {})
if mem:
stats["process_memory_bytes"] = next(iter(mem.values()))
return stats
def extract_certs(current: ParsedMetrics) -> list[dict]:
"""Extract TLS certificate expiry info from traefik_tls_certs_not_after.
Returns a list of dicts:
[{"serial": str, "cn": str, "sans": str, "not_after": datetime}, ...]
"""
certs = []
samples = current.get("traefik_tls_certs_not_after", {})
for labels, ts_value in samples.items():
label_dict = dict(labels)
serial = label_dict.get("serial", "")
if not serial:
continue
try:
not_after = datetime.fromtimestamp(ts_value, tz=timezone.utc)
except (ValueError, OSError, OverflowError):
continue
certs.append({
"serial": serial,
"cn": label_dict.get("cn") or None,
"sans": label_dict.get("sans") or None,
"not_after": not_after,
})
return certs
def _percentile_ms(
buckets: dict[frozenset, float],
service: str,
pct: float,
) -> float:
"""Linear interpolation of a Prometheus histogram percentile, returned in ms."""
router_buckets: list[tuple[float, float]] = []
for labels, count in buckets.items():
label_dict = dict(labels)
if label_dict.get("service") != service:
continue
le_str = label_dict.get("le", "")
try:
le = float("inf") if le_str == "+Inf" else float(le_str)
router_buckets.append((le, count))
except ValueError:
continue
if not router_buckets:
return 0.0
router_buckets.sort(key=lambda t: t[0])
total = router_buckets[-1][1] # +Inf bucket count == total requests
if total == 0.0:
return 0.0
target = total * pct
for i, (le, count) in enumerate(router_buckets):
if count < target:
continue
if i == 0:
ratio = target / count if count > 0 else 0.0
return le * ratio * 1000.0
prev_le, prev_count = router_buckets[i - 1]
if count == prev_count:
return prev_le * 1000.0
if math.isinf(le):
return prev_le * 1000.0
fraction = (target - prev_count) / (count - prev_count)
return (prev_le + fraction * (le - prev_le)) * 1000.0
return 0.0
# ──────────────────────────────────────────────────────────────────────────────
# HTTP fetch
# ──────────────────────────────────────────────────────────────────────────────
async def fetch_metrics(metrics_url: str) -> ParsedMetrics:
"""Fetch and parse Prometheus metrics from the given URL."""
async with httpx.AsyncClient(timeout=10.0) as client:
response = await client.get(metrics_url)
response.raise_for_status()
return parse_prometheus(response.text)
@@ -0,0 +1,31 @@
{# plugins/traefik/templates/traefik/index.html #}
{% extends "base.html" %}
{% block title %}Traefik — Steward{% endblock %}
{% block content %}
<div style="display:flex;align-items:center;justify-content:space-between;flex-wrap:wrap;gap:0.75rem;margin-bottom:1.5rem;">
<div style="display:flex;align-items:baseline;gap:1rem;">
<h1 class="page-title" style="margin-bottom:0;">Traefik Services</h1>
<span style="color:var(--text-muted);font-size:0.85rem;">polling every {{ poll_interval }}s</span>
</div>
{% include "_time_range.html" %}
</div>
<div id="traefik-rows"
hx-get="/plugins/traefik/rows"
hx-trigger="load, every {{ poll_interval }}s, rangeChange from:body"
hx-include="#time-range"
hx-swap="innerHTML">
</div>
{% if access_log_enabled %}
<div style="margin-top:1.5rem;">
<h2 class="section-title" style="margin-bottom:1rem;">Traffic Origin</h2>
<div id="traefik-traffic"
hx-get="/plugins/traefik/traffic"
hx-trigger="load, every {{ poll_interval }}s, rangeChange from:body"
hx-include="#time-range"
hx-swap="innerHTML">
</div>
</div>
{% endif %}
{% endblock %}
+127
View File
@@ -0,0 +1,127 @@
{# plugins/traefik/templates/traefik/rows.html #}
{# ── Global stats bar ──────────────────────────────────────────────────────── #}
{% if global_stat %}
<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--card-bg);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;">
{% if global_stat.open_conns_total is not none %}
<span>
<span style="color:var(--text-muted);">Open conns</span>
<span style="margin-left:0.4rem;font-weight:600;">{{ global_stat.open_conns_total | int }}</span>
</span>
{% endif %}
{% if global_stat.config_last_reload_success is not none %}
<span>
<span style="color:var(--text-muted);">Config</span>
{% if global_stat.config_last_reload_success == 1.0 %}
<span style="margin-left:0.4rem;color:var(--green);">&#10003; OK</span>
{% else %}
<span style="margin-left:0.4rem;color:var(--red);">&#10007; failed</span>
{% endif %}
{% if global_stat.config_reloads_total is not none %}
<span style="color:var(--text-dim);margin-left:0.3rem;">({{ global_stat.config_reloads_total | int }} reloads)</span>
{% endif %}
</span>
{% endif %}
{% if global_stat.process_memory_bytes is not none %}
<span>
<span style="color:var(--text-muted);">Memory</span>
{% set mb = (global_stat.process_memory_bytes / 1048576) %}
<span style="margin-left:0.4rem;">
{% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.0f"|format(mb) }} MB{% endif %}
</span>
</span>
{% endif %}
<span style="margin-left:auto;color:var(--text-dim);font-size:0.78rem;">
{{ global_stat.scraped_at.strftime("%H:%M:%S") }} UTC
</span>
</div>
{% endif %}
{# ── Per-service cards + cert card in shared columns layout ───────────────── #}
{% if router_data or certs %}
<div style="column-width:340px;column-count:3;column-gap:1rem;">
{% for r in router_data %}
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
<h3 class="section-title" style="margin-bottom:0.75rem;word-break:break-all;">{{ r.name }}</h3>
<table class="table" style="font-size:0.85rem;">
<tr>
<td style="color:var(--text-muted);padding:0.2rem 0;">Req/s</td>
<td style="padding:0.2rem 0.5rem;">{{ "%.2f"|format(r.latest.request_rate) }}</td>
<td style="padding:0.2rem 0;">{{ r.sparkline_req | safe }}</td>
</tr>
<tr>
<td style="color:var(--text-muted);padding:0.2rem 0;">Bandwidth</td>
<td style="padding:0.2rem 0.5rem;">
{% set bps = r.latest.response_bytes_rate %}
{% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }} MB/s
{% elif bps >= 1024 %}{{ "%.1f"|format(bps / 1024) }} KB/s
{% else %}{{ "%.0f"|format(bps) }} B/s{% endif %}
</td>
<td style="padding:0.2rem 0;"></td>
</tr>
<tr>
<td style="color:var(--text-muted);padding:0.2rem 0;">p95 ms</td>
<td style="padding:0.2rem 0.5rem;">{{ "%.1f"|format(r.latest.latency_p95_ms) }}</td>
<td style="padding:0.2rem 0;">{{ r.sparkline_p95 | safe }}</td>
</tr>
<tr>
<td style="color:var(--text-muted);padding:0.2rem 0;">p99 ms</td>
<td style="padding:0.2rem 0.5rem;">{{ "%.1f"|format(r.latest.latency_p99_ms) }}</td>
<td style="padding:0.2rem 0;"></td>
</tr>
<tr>
<td style="color:var(--text-muted);padding:0.2rem 0;">4xx %</td>
<td style="color:{% if r.latest.error_rate_4xx_pct > 1 %}var(--yellow){% else %}var(--green){% endif %};padding:0.2rem 0.5rem;">
{{ "%.1f"|format(r.latest.error_rate_4xx_pct) }}%
</td>
<td style="padding:0.2rem 0;"></td>
</tr>
<tr>
<td style="color:var(--text-muted);padding:0.2rem 0;">5xx %</td>
<td style="color:{% if r.latest.error_rate_5xx_pct > 0.1 %}var(--red){% else %}var(--green){% endif %};padding:0.2rem 0.5rem;">
{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}%
</td>
<td style="padding:0.2rem 0;">{{ r.sparkline_5xx | safe }}</td>
</tr>
</table>
<div style="color:var(--text-dim);font-size:0.75rem;margin-top:0.5rem;">
Last scraped {{ r.latest.scraped_at.strftime("%H:%M:%S") }} UTC
</div>
</div>
{% endfor %}
{% if certs %}
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
<div class="section-title" style="margin-bottom:0.75rem;">TLS Certificates</div>
<table class="table" style="font-size:0.83rem;width:100%;">
{% for cert in certs %}
<tr>
<td style="padding:0.2rem 0 0.2rem 0;word-break:break-all;">{{ cert.cn }}</td>
<td style="text-align:right;padding:0.2rem 0;white-space:nowrap;font-weight:600;
{% if cert.days_remaining < 7 %}color:var(--red);
{% elif cert.days_remaining < 30 %}color:var(--yellow);
{% else %}color:var(--green);{% endif %}">
{{ cert.days_remaining | int }}d
</td>
</tr>
<tr>
<td colspan="2" style="padding:0 0 0.4rem 0;color:var(--text-dim);font-size:0.75rem;">
{{ cert.not_after.strftime("%Y-%m-%d") }}
</td>
</tr>
{% endfor %}
</table>
</div>
{% endif %}
</div>
{% else %}
<div class="card">
<p style="color:#606080;">No Traefik metrics yet. Configure <code>metrics_url</code> in <code>config.yaml</code> under <code>plugins.traefik</code>.</p>
</div>
{% endif %}
@@ -0,0 +1,101 @@
{# plugins/traefik/templates/traefik/traffic.html #}
{% if summary %}
{# ── Overview bar ─────────────────────────────────────────────────────────── #}
<div style="display:flex;flex-wrap:wrap;gap:1.5rem;margin-bottom:1.25rem;padding:0.75rem 1rem;background:var(--card-bg);border:1px solid var(--border);border-radius:6px;font-size:0.85rem;font-variant-numeric:tabular-nums;align-items:center;">
<span>
<span style="color:var(--text-muted);">Total</span>
<span style="margin-left:0.4rem;font-weight:600;">{{ summary.total }}</span>
</span>
<span>
<span style="color:var(--text-muted);">Internal</span>
<span style="margin-left:0.4rem;color:var(--green);font-weight:600;">{{ summary.internal }}</span>
</span>
<span>
<span style="color:var(--text-muted);">External</span>
<span style="margin-left:0.4rem;color:{% if summary.external_pct > 50 %}var(--yellow){% else %}var(--green){% endif %};font-weight:600;">
{{ summary.external }} ({{ "%.1f"|format(summary.external_pct) }}%)
</span>
</span>
<span>
<span style="color:var(--text-muted);">Bytes</span>
<span style="margin-left:0.4rem;">
{% set mb = summary.total_bytes / 1048576 %}
{% if mb >= 1024 %}{{ "%.1f"|format(mb / 1024) }} GB{% else %}{{ "%.1f"|format(mb) }} MB{% endif %}
</span>
</span>
{# External % sparkline #}
<span style="margin-left:auto;">
<span style="color:var(--text-muted);font-size:0.78rem;margin-right:0.4rem;">ext % over time</span>
{{ summary.sparkline_external | safe }}
</span>
</div>
{# ── Internal/external fill bar ───────────────────────────────────────────── #}
{% set ext_pct = summary.external_pct %}
<div style="height:6px;border-radius:3px;background:var(--border);margin-bottom:1.25rem;overflow:hidden;">
<div style="height:100%;width:{{ ext_pct | round(1) }}%;background:var(--yellow);border-radius:3px;transition:width 0.4s;"></div>
</div>
{# ── Detail cards in columns ──────────────────────────────────────────────── #}
<div style="column-width:300px;column-count:3;column-gap:1rem;">
{# Top source IPs #}
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
<div class="section-title" style="margin-bottom:0.75rem;">Top Source IPs</div>
<table class="table" style="font-size:0.82rem;width:100%;">
{% for entry in summary.top_ips %}
<tr>
<td style="padding:0.2rem 0.5rem 0.2rem 0;font-family:monospace;">{{ entry.ip }}</td>
{% if entry.country %}
<td style="padding:0.2rem 0.3rem;color:var(--text-muted);">{{ entry.country }}</td>
{% endif %}
<td style="text-align:right;padding:0.2rem 0;">
<span style="color:{% if entry.internal %}var(--text-muted){% else %}var(--yellow){% endif %};">
{{ entry.count }}
</span>
</td>
</tr>
{% endfor %}
</table>
</div>
{# Top routers #}
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
<div class="section-title" style="margin-bottom:0.75rem;">Top Services Hit</div>
<table class="table" style="font-size:0.82rem;width:100%;">
{% for entry in summary.top_routers %}
<tr>
<td style="padding:0.2rem 0.5rem 0.2rem 0;word-break:break-all;">{{ entry.router }}</td>
<td style="text-align:right;padding:0.2rem 0;">{{ entry.count }}</td>
</tr>
{% endfor %}
</table>
</div>
{# Top countries (only shown if GeoIP is active) #}
{% if summary.top_countries %}
<div class="card" style="break-inside:avoid;margin-bottom:1rem;">
<div class="section-title" style="margin-bottom:0.75rem;">External Traffic by Country</div>
<table class="table" style="font-size:0.82rem;width:100%;">
{% for entry in summary.top_countries %}
<tr>
<td style="padding:0.2rem 0.5rem 0.2rem 0;">{{ entry.country }}</td>
<td style="text-align:right;padding:0.2rem 0;">{{ entry.count }}</td>
</tr>
{% endfor %}
</table>
</div>
{% endif %}
</div>
{% else %}
<div class="card" style="color:var(--text-muted);font-size:0.85rem;">
No access log data yet.
{% if not access_log_enabled %}
Enable <code>access_log.enabled: true</code> in your plugin config and bind-mount the Traefik log directory.
{% endif %}
</div>
{% endif %}
@@ -0,0 +1,56 @@
{# plugins/traefik/templates/traefik/widget.html #}
{% if expiring_certs %}
<div style="margin-bottom:0.6rem;">
{% for cert in expiring_certs %}
<div style="display:flex;justify-content:space-between;font-size:0.8rem;padding:0.15rem 0;
{% if cert.days_remaining < 7 %}color:var(--red);{% else %}color:var(--yellow);{% endif %}">
<span>&#9888; {{ cert.cn }}</span>
<span>{{ cert.days_remaining | int }}d</span>
</div>
{% endfor %}
</div>
{% endif %}
{% if router_data %}
{% for r in router_data %}
<div class="ping-row">
<div class="ping-meta">
<span class="ping-name" style="font-size:0.8rem;word-break:break-all;">{{ r.name }}</span>
</div>
<div style="flex:1;display:flex;gap:1.5rem;font-size:0.82rem;font-variant-numeric:tabular-nums;">
<span title="Requests/s">
<span style="color:var(--text-muted);">req/s</span>
<span style="margin-left:0.3rem;">{{ "%.2f"|format(r.latest.request_rate) }}</span>
</span>
<span title="Bandwidth">
<span style="color:var(--text-muted);">bw</span>
<span style="margin-left:0.3rem;">
{% set bps = r.latest.response_bytes_rate %}
{% if bps >= 1048576 %}{{ "%.1f"|format(bps / 1048576) }}MB/s
{% elif bps >= 1024 %}{{ "%.0f"|format(bps / 1024) }}KB/s
{% else %}{{ "%.0f"|format(bps) }}B/s{% endif %}
</span>
</span>
<span title="p95 latency">
<span style="color:var(--text-muted);">p95</span>
<span style="margin-left:0.3rem;
{% if r.latest.latency_p95_ms > 500 %}color:var(--red){% elif r.latest.latency_p95_ms > 200 %}color:var(--yellow){% else %}color:var(--green){% endif %};">
{{ "%.0f"|format(r.latest.latency_p95_ms) }}ms
</span>
</span>
{% if r.latest.error_rate_5xx_pct > 0 %}
<span title="5xx error rate">
<span style="color:var(--red);">{{ "%.1f"|format(r.latest.error_rate_5xx_pct) }}% 5xx</span>
</span>
{% endif %}
</div>
</div>
{% endfor %}
{% if total_routers > 5 %}
<div style="color:var(--text-dim);font-size:0.75rem;padding:0.25rem 0;">
+{{ total_routers - 5 }} more — <a href="/plugins/traefik/">view all →</a>
</div>
{% endif %}
{% else %}
<p class="empty" style="padding:0.5rem 0;font-size:0.85rem;">No Traefik data yet.</p>
{% endif %}
@@ -0,0 +1,40 @@
{# traefik/widget_access_log.html — dashboard widget: traffic origin summary #}
{% if not summary %}
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No access log data yet.</div>
{% else %}
<div style="display:flex;gap:1rem;margin-bottom:0.75rem;flex-wrap:wrap;">
<div>
<span style="font-size:1.4rem;font-weight:700;line-height:1;">{{ summary.total | int }}</span>
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">total req</span>
</div>
<div>
<span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--orange);">
{{ "%.1f" | format(summary.external_pct) }}%
</span>
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">external</span>
</div>
</div>
{% if summary.top_ips %}
<div style="font-size:0.72rem;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.05em;margin-bottom:0.4rem;">
Top IPs
{% if summary.ip_filter != "all" %}
<span style="color:var(--accent);margin-left:0.4rem;">{{ summary.ip_filter }}</span>
{% endif %}
</div>
<div style="display:grid;gap:2px;">
{% for ip in summary.top_ips %}
<div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.2rem 0;">
<span class="dot {% if ip.internal %}dot-dim{% else %}dot-warn{% endif %}"></span>
<span style="flex:1;font-family:ui-monospace,monospace;font-size:0.78rem;
color:{% if ip.internal %}var(--text-muted){% else %}var(--text){% endif %};">
{{ ip.ip }}
</span>
<span style="color:var(--text-muted);font-size:0.78rem;">{{ ip.count }}</span>
</div>
{% endfor %}
</div>
{% endif %}
{% endif %}
@@ -0,0 +1,75 @@
{# traefik/widget_request_chart.html — Chart.js request rate + error % over time #}
{% if not labels_json or labels_json == '[]' %}
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">No data for last {{ hours }}h.</div>
{% else %}
<div style="position:relative;height:160px;">
<canvas id="chart-traefik-req-{{ widget_id }}"></canvas>
</div>
<script>
(function() {
var ctx = document.getElementById("chart-traefik-req-{{ widget_id }}");
if (!ctx || !window.Chart) return;
var existing = Chart.getChart(ctx);
if (existing) existing.destroy();
new Chart(ctx.getContext("2d"), {
type: "line",
data: {
labels: {{ labels_json | safe }},
datasets: [
{
label: "Req/s",
data: {{ req_rate_json | safe }},
borderColor: "#7c5ef4",
backgroundColor: "rgba(124,94,244,0.08)",
fill: true,
tension: 0.3,
pointRadius: 0,
borderWidth: 1.5,
yAxisID: "y",
},
{
label: "5xx %",
data: {{ error_rate_json | safe }},
borderColor: "#e83848",
fill: false,
tension: 0.3,
pointRadius: 0,
borderWidth: 1.5,
yAxisID: "y2",
},
]
},
options: {
responsive: true,
maintainAspectRatio: false,
animation: false,
plugins: {
legend: {
labels: { color: "#6858a0", boxWidth: 10, font: { size: 10 } }
}
},
scales: {
x: { display: false },
y: {
min: 0,
grid: { color: "rgba(30,30,62,0.8)" },
ticks: { color: "#6858a0", font: { size: 10 } },
title: { display: true, text: "req/s", color: "#6858a0", font: { size: 9 } },
},
y2: {
position: "right",
min: 0,
max: 100,
grid: { drawOnChartArea: false },
ticks: { color: "#e83848", font: { size: 10 } },
title: { display: true, text: "5xx%", color: "#e83848", font: { size: 9 } },
},
}
}
});
})();
</script>
<div style="font-size:0.72rem;color:var(--text-muted);margin-top:0.35rem;text-align:right;">
last {{ hours }}h
</div>
{% endif %}
+33
View File
@@ -0,0 +1,33 @@
# plugins/unifi/__init__.py
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from quart import Quart
_app: "Quart | None" = None
def setup(app: "Quart") -> None:
global _app
_app = app
from .models import ( # noqa: F401
UnifiWanStat, UnifiClientSnapshot, UnifiWlanStat, UnifiDpiSnapshot,
UnifiDevice, UnifiKnownClient, UnifiEvent, UnifiAlarm,
UnifiSpeedtestResult, UnifiNetwork, UnifiPortForward, UnifiFwRule,
)
def get_scheduled_tasks() -> list:
from .scheduler import make_poll_task
return [make_poll_task(_app)]
def get_blueprint():
from .routes import unifi_bp
return unifi_bp
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (UniFi network overview).
return [{"label": "UniFi", "href": "/plugins/unifi/"}]
+170
View File
@@ -0,0 +1,170 @@
# plugins/unifi/client.py
"""Async UniFi Network controller API client.
Supports UDM/UDM-Pro (firmware 2+) using the /proxy/network/ path prefix
and Bearer-token auth via the /api/auth/login endpoint.
On 401, re-authenticates automatically once before raising.
"""
from __future__ import annotations
import logging
import httpx
logger = logging.getLogger(__name__)
class UnifiClient:
def __init__(
self,
host: str,
username: str,
password: str,
site: str = "default",
verify_ssl: bool = False,
) -> None:
self._host = host.rstrip("/")
self._username = username
self._password = password
self._site = site
self._verify_ssl = verify_ssl
self._http: httpx.AsyncClient | None = None
self._csrf_token: str = ""
# ── Lifecycle ─────────────────────────────────────────────────────────────
async def _ensure_http(self) -> None:
if self._http is None:
self._http = httpx.AsyncClient(
verify=self._verify_ssl,
timeout=15.0,
follow_redirects=True,
)
async def login(self) -> None:
await self._ensure_http()
resp = await self._http.post(
f"{self._host}/api/auth/login",
json={"username": self._username, "password": self._password, "rememberMe": False},
)
resp.raise_for_status()
self._csrf_token = resp.headers.get("X-CSRF-Token", "")
logger.debug("UniFi login OK (csrf=%s…)", self._csrf_token[:8] if self._csrf_token else "none")
async def close(self) -> None:
if self._http:
await self._http.aclose()
self._http = None
# ── API helpers ───────────────────────────────────────────────────────────
def _api_url(self, path: str) -> str:
return f"{self._host}/proxy/network/api/s/{self._site}{path}"
def _headers(self) -> dict:
return {"X-CSRF-Token": self._csrf_token} if self._csrf_token else {}
async def _get(self, path: str, params: dict | None = None) -> list | dict:
await self._ensure_http()
resp = await self._http.get(self._api_url(path), headers=self._headers(), params=params)
if resp.status_code == 401:
await self.login()
resp = await self._http.get(self._api_url(path), headers=self._headers(), params=params)
resp.raise_for_status()
data = resp.json()
return data.get("data", data)
# ── Core monitoring ───────────────────────────────────────────────────────
async def get_health(self) -> list[dict]:
"""Subsystem health (wan, wlan, lan, www)."""
result = await self._get("/stat/health")
return result if isinstance(result, list) else []
async def get_active_clients(self) -> list[dict]:
"""Currently connected clients."""
result = await self._get("/stat/sta")
return result if isinstance(result, list) else []
async def get_devices(self) -> list[dict]:
"""Managed network devices (APs, switches, gateways) with full detail."""
result = await self._get("/stat/device")
return result if isinstance(result, list) else []
# ── Events & alarms ───────────────────────────────────────────────────────
async def get_events(self, limit: int = 200) -> list[dict]:
"""Recent site events (client connects, IDS alerts, device state changes, etc.)."""
result = await self._get("/stat/event", params={"_limit": limit, "_sort": "-time"})
return result if isinstance(result, list) else []
async def get_alarms(self, archived: bool = False) -> list[dict]:
"""Active (or archived) alarms."""
result = await self._get("/stat/alarm")
if not isinstance(result, list):
return []
return [a for a in result if a.get("archived", False) == archived]
# ── Traffic & DPI ─────────────────────────────────────────────────────────
async def get_dpi_stats(self) -> list[dict]:
"""DPI per-client application/category traffic stats (cumulative counters)."""
result = await self._get("/stat/dpi")
return result if isinstance(result, list) else []
async def get_dpi_site_stats(self) -> list[dict]:
"""DPI site-level category totals (not per-client)."""
result = await self._get("/dpi")
return result if isinstance(result, list) else []
async def get_speedtest_status(self) -> dict | None:
"""Most recent WAN speed test result."""
result = await self._get("/stat/speedtest-status")
if isinstance(result, list) and result:
return result[0]
if isinstance(result, dict):
return result
return None
# ── Network inventory ─────────────────────────────────────────────────────
async def get_known_clients(self) -> list[dict]:
"""All historically seen clients (not just currently connected)."""
result = await self._get("/rest/user")
return result if isinstance(result, list) else []
async def get_networks(self) -> list[dict]:
"""Network/VLAN configurations."""
result = await self._get("/rest/networkconf")
return result if isinstance(result, list) else []
async def get_wlan_configs(self) -> list[dict]:
"""WLAN (SSID) configurations."""
result = await self._get("/rest/wlanconf")
return result if isinstance(result, list) else []
async def get_port_forwards(self) -> list[dict]:
"""Port forwarding rules."""
result = await self._get("/rest/portforwarding")
return result if isinstance(result, list) else []
async def get_firewall_rules(self) -> list[dict]:
"""Firewall rules."""
result = await self._get("/rest/firewallrule")
return result if isinstance(result, list) else []
async def get_firewall_groups(self) -> list[dict]:
"""Firewall groups (IP sets / port sets referenced by rules)."""
result = await self._get("/rest/firewallgroup")
return result if isinstance(result, list) else []
# ── System ────────────────────────────────────────────────────────────────
async def get_sysinfo(self) -> dict | None:
"""Controller system information (version, uptime, hostname)."""
result = await self._get("/stat/sysinfo")
if isinstance(result, list) and result:
return result[0]
if isinstance(result, dict):
return result
return None
+72
View File
@@ -0,0 +1,72 @@
# plugins/unifi/migrations/env.py
"""Alembic env.py for the UniFi plugin (standalone dev use).
At app startup, migration_runner.py uses the core env.py with version_locations.
"""
import asyncio
import os
import sys
from pathlib import Path
from logging.config import fileConfig
from sqlalchemy import pool
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import async_engine_from_config
from alembic import context
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
from steward.models.base import Base
import steward.models # noqa: F401
from plugins.unifi.models import UnifiWanStat, UnifiClientSnapshot, UnifiDevice # noqa: F401
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def _get_url() -> str:
import yaml
cfg_path = os.environ.get("STEWARD_CONFIG", "config.yaml")
try:
with open(cfg_path) as f:
cfg = yaml.safe_load(f) or {}
url = cfg.get("database", {}).get("url", "")
except FileNotFoundError:
url = ""
return os.environ.get("STEWARD_DATABASE__URL", url)
def run_migrations_offline() -> None:
context.configure(
url=_get_url(), target_metadata=target_metadata,
literal_binds=True, dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def do_run_migrations(connection: Connection) -> None:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
cfg = config.get_section(config.config_ini_section, {})
cfg["sqlalchemy.url"] = _get_url()
connectable = async_engine_from_config(cfg, prefix="sqlalchemy.", poolclass=pool.NullPool)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
asyncio.run(run_async_migrations())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
@@ -0,0 +1,62 @@
"""UniFi plugin initial tables
Revision ID: unifi_001_initial
Revises: (none — branch off core via depends_on)
Create Date: 2026-03-20
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "unifi_001_initial"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = "unifi"
depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
def upgrade() -> None:
op.create_table(
"unifi_wan_stats",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("is_up", sa.Boolean, nullable=False, server_default="false"),
sa.Column("wan_ip", sa.String(64), nullable=True),
sa.Column("latency_ms", sa.Float, nullable=True),
sa.Column("rx_bytes_rate", sa.Float, nullable=True),
sa.Column("tx_bytes_rate", sa.Float, nullable=True),
sa.Column("uptime_seconds", sa.Integer, nullable=True),
)
op.create_index("ix_unifi_wan_stats_scraped", "unifi_wan_stats", ["scraped_at"])
op.create_table(
"unifi_client_snapshots",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("total_clients", sa.Integer, nullable=False, server_default="0"),
sa.Column("wireless_clients", sa.Integer, nullable=False, server_default="0"),
sa.Column("wired_clients", sa.Integer, nullable=False, server_default="0"),
sa.Column("guest_clients", sa.Integer, nullable=False, server_default="0"),
sa.Column("top_clients_json", sa.Text, nullable=False, server_default="[]"),
)
op.create_index("ix_unifi_client_snapshots_scraped", "unifi_client_snapshots", ["scraped_at"])
op.create_table(
"unifi_devices",
sa.Column("mac", sa.String(32), primary_key=True),
sa.Column("name", sa.String(255), nullable=True),
sa.Column("model", sa.String(64), nullable=True),
sa.Column("device_type", sa.String(16), nullable=True),
sa.Column("ip", sa.String(64), nullable=True),
sa.Column("state", sa.Integer, nullable=False, server_default="0"),
sa.Column("uptime_seconds", sa.Integer, nullable=True),
sa.Column("last_seen", sa.DateTime(timezone=True), nullable=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
op.drop_table("unifi_devices")
op.drop_index("ix_unifi_client_snapshots_scraped", "unifi_client_snapshots")
op.drop_table("unifi_client_snapshots")
op.drop_index("ix_unifi_wan_stats_scraped", "unifi_wan_stats")
op.drop_table("unifi_wan_stats")
@@ -0,0 +1,156 @@
"""UniFi expanded tables: wlan stats, dpi, events, alarms, known clients,
speedtest results, networks, port forwards, firewall rules.
Also adds missing 'version' column to unifi_devices.
Revision ID: unifi_002_expanded
Revises: unifi_001_initial
Create Date: 2026-03-20
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "unifi_002_expanded"
down_revision: Union[str, None] = "unifi_001_initial"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ── Patch unifi_devices to add missing version column ─────────────────────
op.add_column("unifi_devices", sa.Column("version", sa.String(64), nullable=True))
# ── WLAN stats (time-series per SSID+radio) ───────────────────────────────
op.create_table(
"unifi_wlan_stats",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("ssid", sa.String(255), nullable=False),
sa.Column("radio", sa.String(8), nullable=True),
sa.Column("num_sta", sa.Integer, nullable=False, server_default="0"),
sa.Column("tx_bytes", sa.Float, nullable=True),
sa.Column("rx_bytes", sa.Float, nullable=True),
sa.Column("channel", sa.Integer, nullable=True),
sa.Column("satisfaction", sa.Integer, nullable=True),
)
op.create_index("ix_unifi_wlan_stats_scraped", "unifi_wlan_stats", ["scraped_at"])
# ── DPI snapshots (site-level category totals) ────────────────────────────
op.create_table(
"unifi_dpi_snapshots",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("by_category_json", sa.Text, nullable=False, server_default="[]"),
sa.Column("by_client_json", sa.Text, nullable=False, server_default="[]"),
)
op.create_index("ix_unifi_dpi_scraped", "unifi_dpi_snapshots", ["scraped_at"])
# ── Known clients (upserted by MAC) ───────────────────────────────────────
op.create_table(
"unifi_known_clients",
sa.Column("mac", sa.String(32), primary_key=True),
sa.Column("hostname", sa.String(255), nullable=True),
sa.Column("name", sa.String(255), nullable=True),
sa.Column("ip", sa.String(64), nullable=True),
sa.Column("mac_vendor", sa.String(128), nullable=True),
sa.Column("note", sa.Text, nullable=True),
sa.Column("is_blocked", sa.Boolean, nullable=False, server_default="false"),
sa.Column("first_seen", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_seen", sa.DateTime(timezone=True), nullable=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
)
# ── Events (upserted by UniFi event ID) ───────────────────────────────────
op.create_table(
"unifi_events",
sa.Column("unifi_id", sa.String(64), primary_key=True),
sa.Column("event_time", sa.DateTime(timezone=True), nullable=False),
sa.Column("event_key", sa.String(64), nullable=False),
sa.Column("category", sa.String(16), nullable=False),
sa.Column("message", sa.Text, nullable=False),
sa.Column("source_mac", sa.String(32), nullable=True),
sa.Column("source_ip", sa.String(64), nullable=True),
sa.Column("details_json", sa.Text, nullable=False, server_default="{}"),
)
op.create_index("ix_unifi_events_time", "unifi_events", ["event_time"])
# ── Alarms (upserted by UniFi alarm ID) ───────────────────────────────────
op.create_table(
"unifi_alarms",
sa.Column("unifi_id", sa.String(64), primary_key=True),
sa.Column("alarm_time", sa.DateTime(timezone=True), nullable=False),
sa.Column("alarm_key", sa.String(64), nullable=False),
sa.Column("message", sa.Text, nullable=False),
sa.Column("archived", sa.Boolean, nullable=False, server_default="false"),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_unifi_alarms_time", "unifi_alarms", ["alarm_time"])
# ── Speedtest results (upserted by run timestamp) ─────────────────────────
op.create_table(
"unifi_speedtest_results",
sa.Column("run_at", sa.DateTime(timezone=True), primary_key=True),
sa.Column("download_mbps", sa.Float, nullable=True),
sa.Column("upload_mbps", sa.Float, nullable=True),
sa.Column("latency_ms", sa.Float, nullable=True),
sa.Column("server_name", sa.String(255), nullable=True),
)
# ── Networks/VLANs (upserted by UniFi ID) ────────────────────────────────
op.create_table(
"unifi_networks",
sa.Column("unifi_id", sa.String(64), primary_key=True),
sa.Column("name", sa.String(255), nullable=False),
sa.Column("purpose", sa.String(32), nullable=True),
sa.Column("vlan", sa.Integer, nullable=True),
sa.Column("ip_subnet", sa.String(64), nullable=True),
sa.Column("dhcp_enabled", sa.Boolean, nullable=False, server_default="false"),
sa.Column("is_guest", sa.Boolean, nullable=False, server_default="false"),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
)
# ── Port forwards (upserted by UniFi ID) ─────────────────────────────────
op.create_table(
"unifi_port_forwards",
sa.Column("unifi_id", sa.String(64), primary_key=True),
sa.Column("name", sa.String(255), nullable=True),
sa.Column("proto", sa.String(8), nullable=False, server_default="tcp"),
sa.Column("dst_port", sa.String(32), nullable=False),
sa.Column("fwd_ip", sa.String(64), nullable=False),
sa.Column("fwd_port", sa.String(32), nullable=False),
sa.Column("src_filter", sa.String(255), nullable=True),
sa.Column("enabled", sa.Boolean, nullable=False, server_default="true"),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
)
# ── Firewall rules (upserted by UniFi ID) ────────────────────────────────
op.create_table(
"unifi_fw_rules",
sa.Column("unifi_id", sa.String(64), primary_key=True),
sa.Column("name", sa.String(255), nullable=False),
sa.Column("ruleset", sa.String(16), nullable=False),
sa.Column("rule_index", sa.Integer, nullable=True),
sa.Column("action", sa.String(16), nullable=False),
sa.Column("protocol", sa.String(16), nullable=True),
sa.Column("enabled", sa.Boolean, nullable=False, server_default="true"),
sa.Column("src_address", sa.String(255), nullable=True),
sa.Column("dst_address", sa.String(255), nullable=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
op.drop_table("unifi_fw_rules")
op.drop_table("unifi_port_forwards")
op.drop_table("unifi_networks")
op.drop_table("unifi_speedtest_results")
op.drop_index("ix_unifi_alarms_time", "unifi_alarms")
op.drop_table("unifi_alarms")
op.drop_index("ix_unifi_events_time", "unifi_events")
op.drop_table("unifi_events")
op.drop_table("unifi_known_clients")
op.drop_index("ix_unifi_dpi_scraped", "unifi_dpi_snapshots")
op.drop_table("unifi_dpi_snapshots")
op.drop_index("ix_unifi_wlan_stats_scraped", "unifi_wlan_stats")
op.drop_table("unifi_wlan_stats")
op.drop_column("unifi_devices", "version")
+182
View File
@@ -0,0 +1,182 @@
# plugins/unifi/models.py
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from steward.models.base import Base
# ── Time-series (one row per scrape) ──────────────────────────────────────────
class UnifiWanStat(Base):
"""WAN interface health — one row per scrape."""
__tablename__ = "unifi_wan_stats"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
wan_ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
rx_bytes_rate: Mapped[float | None] = mapped_column(Float, nullable=True)
tx_bytes_rate: Mapped[float | None] = mapped_column(Float, nullable=True)
uptime_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
class UnifiClientSnapshot(Base):
"""Aggregated client counts + top talkers — one row per scrape."""
__tablename__ = "unifi_client_snapshots"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
total_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
wireless_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
wired_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
guest_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
top_clients_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
# JSON: [{"mac","hostname","ip","type","tx_rate","rx_rate","signal","essid"}]
class UnifiWlanStat(Base):
"""Per-SSID/radio stats — one row per scrape per SSID+radio combination."""
__tablename__ = "unifi_wlan_stats"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
ssid: Mapped[str] = mapped_column(String(255), nullable=False)
radio: Mapped[str | None] = mapped_column(String(8), nullable=True) # ng=2.4GHz, na=5GHz, 6e=6GHz
num_sta: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
tx_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)
rx_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)
channel: Mapped[int | None] = mapped_column(Integer, nullable=True)
satisfaction: Mapped[int | None] = mapped_column(Integer, nullable=True) # 0-100
class UnifiDpiSnapshot(Base):
"""DPI application/category traffic snapshot — one row per scrape (JSON blob)."""
__tablename__ = "unifi_dpi_snapshots"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
# JSON: [{"cat_id", "cat_name", "rx_bytes", "tx_bytes"}] aggregated across all clients
by_category_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
# JSON: [{"mac", "hostname", "top_cats": [{"cat_name", "bytes"}]}]
by_client_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
# ── Upserted inventory (keyed by UniFi ID or MAC) ─────────────────────────────
class UnifiDevice(Base):
"""Network infrastructure devices — upserted by MAC each scrape."""
__tablename__ = "unifi_devices"
mac: Mapped[str] = mapped_column(String(32), primary_key=True)
name: Mapped[str | None] = mapped_column(String(255), nullable=True)
model: Mapped[str | None] = mapped_column(String(64), nullable=True)
device_type: Mapped[str | None] = mapped_column(String(16), nullable=True)
ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
state: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
uptime_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
version: Mapped[str | None] = mapped_column(String(64), nullable=True)
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
class UnifiKnownClient(Base):
"""All historically seen network clients — upserted by MAC."""
__tablename__ = "unifi_known_clients"
mac: Mapped[str] = mapped_column(String(32), primary_key=True)
hostname: Mapped[str | None] = mapped_column(String(255), nullable=True)
name: Mapped[str | None] = mapped_column(String(255), nullable=True) # user-set alias
ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
mac_vendor: Mapped[str | None] = mapped_column(String(128), nullable=True)
note: Mapped[str | None] = mapped_column(Text, nullable=True)
is_blocked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
first_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
class UnifiEvent(Base):
"""Site event log — upserted by UniFi event ID to avoid duplicates."""
__tablename__ = "unifi_events"
unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
event_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
event_key: Mapped[str] = mapped_column(String(64), nullable=False) # e.g. EVT_WU_Connected
category: Mapped[str] = mapped_column(String(16), nullable=False) # wlan, wired, ap, ids, wan, system
message: Mapped[str] = mapped_column(Text, nullable=False)
source_mac: Mapped[str | None] = mapped_column(String(32), nullable=True)
source_ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
details_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
class UnifiAlarm(Base):
"""Active alarms — upserted by UniFi alarm ID."""
__tablename__ = "unifi_alarms"
unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
alarm_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
alarm_key: Mapped[str] = mapped_column(String(64), nullable=False)
message: Mapped[str] = mapped_column(Text, nullable=False)
archived: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
class UnifiSpeedtestResult(Base):
"""WAN speed test results — upserted by run timestamp."""
__tablename__ = "unifi_speedtest_results"
run_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), primary_key=True)
download_mbps: Mapped[float | None] = mapped_column(Float, nullable=True)
upload_mbps: Mapped[float | None] = mapped_column(Float, nullable=True)
latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
server_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
# ── Config inventory (upserted by UniFi ID) ───────────────────────────────────
class UnifiNetwork(Base):
"""VLAN/network configurations — upserted by UniFi ID."""
__tablename__ = "unifi_networks"
unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(255), nullable=False)
purpose: Mapped[str | None] = mapped_column(String(32), nullable=True) # corporate, guest, vlan-only
vlan: Mapped[int | None] = mapped_column(Integer, nullable=True)
ip_subnet: Mapped[str | None] = mapped_column(String(64), nullable=True)
dhcp_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
is_guest: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
class UnifiPortForward(Base):
"""Port forwarding rules — upserted by UniFi ID."""
__tablename__ = "unifi_port_forwards"
unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str | None] = mapped_column(String(255), nullable=True)
proto: Mapped[str] = mapped_column(String(8), nullable=False, default="tcp")
dst_port: Mapped[str] = mapped_column(String(32), nullable=False)
fwd_ip: Mapped[str] = mapped_column(String(64), nullable=False)
fwd_port: Mapped[str] = mapped_column(String(32), nullable=False)
src_filter: Mapped[str | None] = mapped_column(String(255), nullable=True)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
class UnifiFwRule(Base):
"""Firewall rules — upserted by UniFi ID."""
__tablename__ = "unifi_fw_rules"
unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(255), nullable=False)
ruleset: Mapped[str] = mapped_column(String(16), nullable=False) # WAN_IN, LAN_IN, etc.
rule_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
action: Mapped[str] = mapped_column(String(16), nullable=False) # accept, drop, reject
protocol: Mapped[str | None] = mapped_column(String(16), nullable=True)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
src_address: Mapped[str | None] = mapped_column(String(255), nullable=True)
dst_address: Mapped[str | None] = mapped_column(String(255), nullable=True)
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
+21
View File
@@ -0,0 +1,21 @@
name: unifi
version: "1.0.0"
description: "UniFi Network controller integration — WAN health, devices, clients, DPI"
author: "Steward"
license: "MIT"
min_app_version: "0.1.0"
repository_url: "https://git.fabledsword.com/bvandeusen/Steward-plugins"
homepage: "https://git.fabledsword.com/bvandeusen/Steward-plugins/src/branch/main/unifi"
tags:
- network
- unifi
- ubiquiti
config:
host: "https://192.168.1.1"
username: "admin"
password: ""
site: "default"
verify_ssl: false
poll_interval_seconds: 60
top_clients: 10 # number of top-talker clients to store per snapshot
+356
View File
@@ -0,0 +1,356 @@
# plugins/unifi/routes.py
from __future__ import annotations
import json
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import Integer, cast, func, select
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds
from .models import (
UnifiAlarm, UnifiClientSnapshot, UnifiDevice, UnifiDpiSnapshot,
UnifiEvent, UnifiFwRule, UnifiKnownClient, UnifiNetwork,
UnifiPortForward, UnifiSpeedtestResult, UnifiWanStat, UnifiWlanStat,
)
unifi_bp = Blueprint("unifi", __name__, template_folder="templates")
def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
if len(values) < 2:
return f'<svg width="{width}" height="{height}"></svg>'
mn, mx = min(values), max(values)
if mx == mn:
mx = mn + 1.0
step = width / (len(values) - 1)
pts = []
for i, v in enumerate(values):
x = i * step
y = height - (v - mn) / (mx - mn) * (height - 2) - 1
pts.append(f"{x:.1f},{y:.1f}")
poly = " ".join(pts)
return (
f'<svg width="{width}" height="{height}" viewBox="0 0 {width} {height}" '
f'style="vertical-align:middle;">'
f'<polyline points="{poly}" fill="none" stroke="#6060c0" stroke-width="1.5"/>'
f'</svg>'
)
@unifi_bp.get("/")
@require_role(UserRole.viewer)
async def index():
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
current_range = request.args.get("range", DEFAULT_RANGE)
return await render_template(
"unifi/index.html",
poll_interval=poll_interval,
current_range=current_range,
)
@unifi_bp.get("/rows")
@require_role(UserRole.viewer)
async def rows():
"""HTMX fragment: WAN stats, devices, clients, WLAN — scoped to selected range."""
since, range_key = parse_range(request.args.get("range"))
b_secs = bucket_seconds(since)
async with current_app.db_sessionmaker() as db:
# WAN history — bucketed for accurate full-range sparklines
wan_bucket = (
cast(func.strftime('%s', UnifiWanStat.scraped_at), Integer) / b_secs
).label("bucket")
result = await db.execute(
select(
func.avg(UnifiWanStat.latency_ms).label("latency_ms"),
func.avg(UnifiWanStat.rx_bytes_rate).label("rx_bytes_rate"),
func.avg(UnifiWanStat.tx_bytes_rate).label("tx_bytes_rate"),
func.min(UnifiWanStat.scraped_at).label("scraped_at"),
wan_bucket,
)
.where(UnifiWanStat.scraped_at >= since)
.group_by(wan_bucket)
.order_by(wan_bucket)
)
wan_history = result.all()
# Latest WAN — always most recent raw row for current displayed values
result = await db.execute(
select(UnifiWanStat).order_by(UnifiWanStat.scraped_at.desc()).limit(1)
)
latest_wan = result.scalar_one_or_none()
# Latest client snapshot (always most recent)
result = await db.execute(
select(UnifiClientSnapshot)
.order_by(UnifiClientSnapshot.scraped_at.desc())
.limit(1)
)
latest_clients = result.scalar_one_or_none()
# All devices
result = await db.execute(
select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name)
)
devices = result.scalars().all()
# Client count history — bucketed
cs_bucket = (
cast(func.strftime('%s', UnifiClientSnapshot.scraped_at), Integer) / b_secs
).label("bucket")
result = await db.execute(
select(
func.avg(UnifiClientSnapshot.total_clients).label("total_clients"),
func.min(UnifiClientSnapshot.scraped_at).label("scraped_at"),
cs_bucket,
)
.where(UnifiClientSnapshot.scraped_at >= since)
.group_by(cs_bucket)
.order_by(cs_bucket)
)
client_history = result.all()
# Latest WLAN stats (always most recent per SSID)
result = await db.execute(
select(UnifiWlanStat.ssid).distinct().order_by(UnifiWlanStat.ssid)
)
ssids = [row[0] for row in result.all()]
wlan_latest: list[UnifiWlanStat] = []
for ssid in ssids:
result = await db.execute(
select(UnifiWlanStat)
.where(UnifiWlanStat.ssid == ssid)
.order_by(UnifiWlanStat.scraped_at.desc())
.limit(1)
)
w = result.scalar_one_or_none()
if w:
wlan_latest.append(w)
# Latest speedtest
result = await db.execute(
select(UnifiSpeedtestResult)
.order_by(UnifiSpeedtestResult.run_at.desc())
.limit(1)
)
speedtest = result.scalar_one_or_none()
top_clients = json.loads(latest_clients.top_clients_json) if latest_clients else []
sparkline_latency = _sparkline([r.latency_ms or 0 for r in wan_history])
sparkline_rx = _sparkline([r.rx_bytes_rate or 0 for r in wan_history])
sparkline_tx = _sparkline([r.tx_bytes_rate or 0 for r in wan_history])
sparkline_clients = _sparkline([r.total_clients for r in client_history])
return await render_template(
"unifi/rows.html",
latest_wan=latest_wan,
latest_clients=latest_clients,
devices=devices,
top_clients=top_clients,
wlan_latest=wlan_latest,
speedtest=speedtest,
sparkline_latency=sparkline_latency,
sparkline_rx=sparkline_rx,
sparkline_tx=sparkline_tx,
sparkline_clients=sparkline_clients,
range_key=range_key,
)
@unifi_bp.get("/events")
@require_role(UserRole.viewer)
async def events():
"""HTMX fragment: site events within selected range."""
since, range_key = parse_range(request.args.get("range"))
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(UnifiEvent)
.where(UnifiEvent.event_time >= since)
.order_by(UnifiEvent.event_time.desc())
.limit(500)
)
event_rows = result.scalars().all()
return await render_template(
"unifi/events.html", events=event_rows, range_key=range_key
)
@unifi_bp.get("/dpi")
@require_role(UserRole.viewer)
async def dpi():
"""HTMX fragment: DPI traffic breakdown (latest snapshot)."""
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(UnifiDpiSnapshot)
.order_by(UnifiDpiSnapshot.scraped_at.desc())
.limit(1)
)
snapshot = result.scalar_one_or_none()
categories = []
top_clients = []
if snapshot:
categories = json.loads(snapshot.by_category_json)
top_clients = json.loads(snapshot.by_client_json)
return await render_template(
"unifi/dpi.html", categories=categories, top_clients=top_clients, snapshot=snapshot
)
@unifi_bp.get("/security")
@require_role(UserRole.viewer)
async def security():
"""HTMX fragment: active and archived alarms."""
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(UnifiAlarm)
.where(UnifiAlarm.archived == False) # noqa: E712
.order_by(UnifiAlarm.alarm_time.desc())
)
active_alarms = result.scalars().all()
result = await db.execute(
select(UnifiAlarm)
.where(UnifiAlarm.archived == True) # noqa: E712
.order_by(UnifiAlarm.alarm_time.desc())
.limit(20)
)
archived_alarms = result.scalars().all()
return await render_template(
"unifi/security.html",
active_alarms=active_alarms,
archived_alarms=archived_alarms,
)
@unifi_bp.get("/inventory")
@require_role(UserRole.viewer)
async def inventory():
"""HTMX fragment: known clients, networks, port forwards, firewall rules."""
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(UnifiKnownClient)
.order_by(UnifiKnownClient.last_seen.desc().nullslast())
)
known_clients = result.scalars().all()
result = await db.execute(
select(UnifiNetwork).order_by(UnifiNetwork.name)
)
networks = result.scalars().all()
result = await db.execute(
select(UnifiPortForward).order_by(UnifiPortForward.name)
)
port_forwards = result.scalars().all()
result = await db.execute(
select(UnifiFwRule)
.order_by(UnifiFwRule.ruleset, UnifiFwRule.rule_index)
)
fw_rules = result.scalars().all()
return await render_template(
"unifi/inventory.html",
known_clients=known_clients,
networks=networks,
port_forwards=port_forwards,
fw_rules=fw_rules,
)
@unifi_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""HTMX dashboard widget fragment."""
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(UnifiWanStat).order_by(UnifiWanStat.scraped_at.desc()).limit(1)
)
latest_wan = result.scalar_one_or_none()
result = await db.execute(
select(UnifiClientSnapshot).order_by(UnifiClientSnapshot.scraped_at.desc()).limit(1)
)
latest_clients = result.scalar_one_or_none()
result = await db.execute(select(UnifiDevice))
devices = result.scalars().all()
result = await db.execute(
select(UnifiAlarm).where(UnifiAlarm.archived == False) # noqa: E712
)
active_alarms = result.scalars().all()
offline_devices = [d for d in devices if d.state != 1]
top_clients = json.loads(latest_clients.top_clients_json)[:5] if latest_clients else []
return await render_template(
"unifi/widget.html",
latest_wan=latest_wan,
latest_clients=latest_clients,
offline_devices=offline_devices,
active_alarms=active_alarms,
top_clients=top_clients,
)
@unifi_bp.get("/widget/clients")
@require_role(UserRole.viewer)
async def widget_clients():
"""HTMX dashboard widget: top active clients."""
limit = max(1, min(20, int(request.args.get("limit", 5) or 5)))
widget_id = request.args.get("wid", "0")
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(UnifiClientSnapshot)
.order_by(UnifiClientSnapshot.scraped_at.desc())
.limit(1)
)
snapshot = result.scalar_one_or_none()
clients = json.loads(snapshot.top_clients_json)[:limit] if snapshot else []
return await render_template(
"unifi/widget_clients.html",
clients=clients,
snapshot=snapshot,
widget_id=widget_id,
)
@unifi_bp.get("/widget/devices")
@require_role(UserRole.viewer)
async def widget_devices():
"""HTMX dashboard widget: infrastructure devices with optional type filter."""
type_filter = request.args.get("type_filter", "all")
widget_id = request.args.get("wid", "0")
async with current_app.db_sessionmaker() as db:
result = await db.execute(
select(UnifiDevice).order_by(UnifiDevice.state.desc(), UnifiDevice.name)
)
all_devices = result.scalars().all()
if type_filter == "wireless":
devices = [d for d in all_devices if d.device_type == "uap"]
elif type_filter == "wired":
devices = [d for d in all_devices if d.device_type != "uap"]
elif type_filter == "offline":
devices = [d for d in all_devices if d.state != 1]
else:
devices = list(all_devices)
return await render_template(
"unifi/widget_devices.html",
devices=devices,
type_filter=type_filter,
widget_id=widget_id,
)

Some files were not shown because too many files have changed in this diff Show More